brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52 3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 %s -o - | FileCheck %s4 5; Testing codegen for memcpy with vector operands for all combinations of the following parameters:6;     destination address space: 0, 1, 3, 57;     source address space: 0, 1, 3, 4, 58;     alignment: 1, 2, 8, 169;     sizes: 16, 31, 3210 11 12define void @memcpy_p0_p0_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {13; CHECK-LABEL: memcpy_p0_p0_sz16_align_1_1:14; CHECK:       ; %bb.0: ; %entry15; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]17; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)18; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]19; CHECK-NEXT:    s_waitcnt lgkmcnt(0)20; CHECK-NEXT:    s_setpc_b64 s[30:31]21entry:22  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 16, i1 false)23  ret void24}25 26define void @memcpy_p0_p0_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {27; CHECK-LABEL: memcpy_p0_p0_sz31_align_1_1:28; CHECK:       ; %bb.0: ; %entry29; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)30; CHECK-NEXT:    s_clause 0x231; CHECK-NEXT:    flat_load_dwordx2 v[8:9], v[2:3] offset:2332; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3]33; CHECK-NEXT:    flat_load_dwordx2 v[2:3], v[2:3] offset:1634; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)35; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[8:9] offset:2336; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)37; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]38; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)39; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:1640; CHECK-NEXT:    s_waitcnt lgkmcnt(0)41; CHECK-NEXT:    s_setpc_b64 s[30:31]42entry:43  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 31, i1 false)44  ret void45}46 47define void @memcpy_p0_p0_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {48; CHECK-LABEL: memcpy_p0_p0_sz32_align_1_1:49; CHECK:       ; %bb.0: ; %entry50; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)51; CHECK-NEXT:    s_clause 0x152; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:1653; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]54; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)55; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:1656; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)57; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]58; CHECK-NEXT:    s_waitcnt lgkmcnt(0)59; CHECK-NEXT:    s_setpc_b64 s[30:31]60entry:61  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)62  ret void63}64 65define void @memcpy_p0_p0_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {66; CHECK-LABEL: memcpy_p0_p0_sz16_align_2_2:67; CHECK:       ; %bb.0: ; %entry68; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)69; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]70; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)71; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]72; CHECK-NEXT:    s_waitcnt lgkmcnt(0)73; CHECK-NEXT:    s_setpc_b64 s[30:31]74entry:75  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 16, i1 false)76  ret void77}78 79define void @memcpy_p0_p0_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {80; CHECK-LABEL: memcpy_p0_p0_sz31_align_2_2:81; CHECK:       ; %bb.0: ; %entry82; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)83; CHECK-NEXT:    s_clause 0x284; CHECK-NEXT:    flat_load_dwordx2 v[8:9], v[2:3] offset:2385; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3]86; CHECK-NEXT:    flat_load_dwordx2 v[2:3], v[2:3] offset:1687; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)88; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[8:9] offset:2389; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)90; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]91; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)92; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:1693; CHECK-NEXT:    s_waitcnt lgkmcnt(0)94; CHECK-NEXT:    s_setpc_b64 s[30:31]95entry:96  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 31, i1 false)97  ret void98}99 100define void @memcpy_p0_p0_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {101; CHECK-LABEL: memcpy_p0_p0_sz32_align_2_2:102; CHECK:       ; %bb.0: ; %entry103; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)104; CHECK-NEXT:    s_clause 0x1105; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16106; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]107; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)108; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16109; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)110; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]111; CHECK-NEXT:    s_waitcnt lgkmcnt(0)112; CHECK-NEXT:    s_setpc_b64 s[30:31]113entry:114  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)115  ret void116}117 118define void @memcpy_p0_p0_sz16_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {119; CHECK-LABEL: memcpy_p0_p0_sz16_align_8_8:120; CHECK:       ; %bb.0: ; %entry121; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)122; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]123; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)124; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]125; CHECK-NEXT:    s_waitcnt lgkmcnt(0)126; CHECK-NEXT:    s_setpc_b64 s[30:31]127entry:128  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 16, i1 false)129  ret void130}131 132define void @memcpy_p0_p0_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {133; CHECK-LABEL: memcpy_p0_p0_sz31_align_8_8:134; CHECK:       ; %bb.0: ; %entry135; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)136; CHECK-NEXT:    s_clause 0x1137; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:15138; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]139; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)140; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15141; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)142; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]143; CHECK-NEXT:    s_waitcnt lgkmcnt(0)144; CHECK-NEXT:    s_setpc_b64 s[30:31]145entry:146  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 31, i1 false)147  ret void148}149 150define void @memcpy_p0_p0_sz32_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {151; CHECK-LABEL: memcpy_p0_p0_sz32_align_8_8:152; CHECK:       ; %bb.0: ; %entry153; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)154; CHECK-NEXT:    s_clause 0x1155; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16156; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]157; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)158; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16159; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)160; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]161; CHECK-NEXT:    s_waitcnt lgkmcnt(0)162; CHECK-NEXT:    s_setpc_b64 s[30:31]163entry:164  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 32, i1 false)165  ret void166}167 168define void @memcpy_p0_p0_sz16_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {169; CHECK-LABEL: memcpy_p0_p0_sz16_align_16_16:170; CHECK:       ; %bb.0: ; %entry171; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)172; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]173; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)174; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]175; CHECK-NEXT:    s_waitcnt lgkmcnt(0)176; CHECK-NEXT:    s_setpc_b64 s[30:31]177entry:178  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 16, i1 false)179  ret void180}181 182define void @memcpy_p0_p0_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {183; CHECK-LABEL: memcpy_p0_p0_sz31_align_16_16:184; CHECK:       ; %bb.0: ; %entry185; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)186; CHECK-NEXT:    s_clause 0x1187; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:15188; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]189; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)190; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15191; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)192; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]193; CHECK-NEXT:    s_waitcnt lgkmcnt(0)194; CHECK-NEXT:    s_setpc_b64 s[30:31]195entry:196  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 31, i1 false)197  ret void198}199 200define void @memcpy_p0_p0_sz32_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {201; CHECK-LABEL: memcpy_p0_p0_sz32_align_16_16:202; CHECK:       ; %bb.0: ; %entry203; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)204; CHECK-NEXT:    s_clause 0x1205; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:16206; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]207; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)208; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16209; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)210; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]211; CHECK-NEXT:    s_waitcnt lgkmcnt(0)212; CHECK-NEXT:    s_setpc_b64 s[30:31]213entry:214  tail call void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 32, i1 false)215  ret void216}217 218define void @memcpy_p0_p1_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {219; CHECK-LABEL: memcpy_p0_p1_sz16_align_1_1:220; CHECK:       ; %bb.0: ; %entry221; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)222; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off223; CHECK-NEXT:    s_waitcnt vmcnt(0)224; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]225; CHECK-NEXT:    s_waitcnt lgkmcnt(0)226; CHECK-NEXT:    s_setpc_b64 s[30:31]227entry:228  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 16, i1 false)229  ret void230}231 232define void @memcpy_p0_p1_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {233; CHECK-LABEL: memcpy_p0_p1_sz31_align_1_1:234; CHECK:       ; %bb.0: ; %entry235; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)236; CHECK-NEXT:    s_clause 0x2237; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off offset:23238; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off239; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:16240; CHECK-NEXT:    s_waitcnt vmcnt(2)241; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[8:9] offset:23242; CHECK-NEXT:    s_waitcnt vmcnt(1)243; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]244; CHECK-NEXT:    s_waitcnt vmcnt(0)245; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:16246; CHECK-NEXT:    s_waitcnt lgkmcnt(0)247; CHECK-NEXT:    s_setpc_b64 s[30:31]248entry:249  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 31, i1 false)250  ret void251}252 253define void @memcpy_p0_p1_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {254; CHECK-LABEL: memcpy_p0_p1_sz32_align_1_1:255; CHECK:       ; %bb.0: ; %entry256; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)257; CHECK-NEXT:    s_clause 0x1258; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16259; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off260; CHECK-NEXT:    s_waitcnt vmcnt(1)261; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16262; CHECK-NEXT:    s_waitcnt vmcnt(0)263; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]264; CHECK-NEXT:    s_waitcnt lgkmcnt(0)265; CHECK-NEXT:    s_setpc_b64 s[30:31]266entry:267  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 32, i1 false)268  ret void269}270 271define void @memcpy_p0_p1_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {272; CHECK-LABEL: memcpy_p0_p1_sz16_align_2_2:273; CHECK:       ; %bb.0: ; %entry274; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)275; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off276; CHECK-NEXT:    s_waitcnt vmcnt(0)277; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]278; CHECK-NEXT:    s_waitcnt lgkmcnt(0)279; CHECK-NEXT:    s_setpc_b64 s[30:31]280entry:281  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 16, i1 false)282  ret void283}284 285define void @memcpy_p0_p1_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {286; CHECK-LABEL: memcpy_p0_p1_sz31_align_2_2:287; CHECK:       ; %bb.0: ; %entry288; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)289; CHECK-NEXT:    s_clause 0x2290; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off offset:23291; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off292; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:16293; CHECK-NEXT:    s_waitcnt vmcnt(2)294; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[8:9] offset:23295; CHECK-NEXT:    s_waitcnt vmcnt(1)296; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]297; CHECK-NEXT:    s_waitcnt vmcnt(0)298; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:16299; CHECK-NEXT:    s_waitcnt lgkmcnt(0)300; CHECK-NEXT:    s_setpc_b64 s[30:31]301entry:302  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 31, i1 false)303  ret void304}305 306define void @memcpy_p0_p1_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {307; CHECK-LABEL: memcpy_p0_p1_sz32_align_2_2:308; CHECK:       ; %bb.0: ; %entry309; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)310; CHECK-NEXT:    s_clause 0x1311; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16312; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off313; CHECK-NEXT:    s_waitcnt vmcnt(1)314; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16315; CHECK-NEXT:    s_waitcnt vmcnt(0)316; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]317; CHECK-NEXT:    s_waitcnt lgkmcnt(0)318; CHECK-NEXT:    s_setpc_b64 s[30:31]319entry:320  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 32, i1 false)321  ret void322}323 324define void @memcpy_p0_p1_sz16_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {325; CHECK-LABEL: memcpy_p0_p1_sz16_align_8_8:326; CHECK:       ; %bb.0: ; %entry327; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)328; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off329; CHECK-NEXT:    s_waitcnt vmcnt(0)330; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]331; CHECK-NEXT:    s_waitcnt lgkmcnt(0)332; CHECK-NEXT:    s_setpc_b64 s[30:31]333entry:334  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 16, i1 false)335  ret void336}337 338define void @memcpy_p0_p1_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {339; CHECK-LABEL: memcpy_p0_p1_sz31_align_8_8:340; CHECK:       ; %bb.0: ; %entry341; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)342; CHECK-NEXT:    s_clause 0x1343; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:15344; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off345; CHECK-NEXT:    s_waitcnt vmcnt(1)346; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15347; CHECK-NEXT:    s_waitcnt vmcnt(0)348; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]349; CHECK-NEXT:    s_waitcnt lgkmcnt(0)350; CHECK-NEXT:    s_setpc_b64 s[30:31]351entry:352  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 31, i1 false)353  ret void354}355 356define void @memcpy_p0_p1_sz32_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {357; CHECK-LABEL: memcpy_p0_p1_sz32_align_8_8:358; CHECK:       ; %bb.0: ; %entry359; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)360; CHECK-NEXT:    s_clause 0x1361; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16362; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off363; CHECK-NEXT:    s_waitcnt vmcnt(1)364; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16365; CHECK-NEXT:    s_waitcnt vmcnt(0)366; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]367; CHECK-NEXT:    s_waitcnt lgkmcnt(0)368; CHECK-NEXT:    s_setpc_b64 s[30:31]369entry:370  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 32, i1 false)371  ret void372}373 374define void @memcpy_p0_p1_sz16_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {375; CHECK-LABEL: memcpy_p0_p1_sz16_align_16_16:376; CHECK:       ; %bb.0: ; %entry377; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)378; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off379; CHECK-NEXT:    s_waitcnt vmcnt(0)380; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]381; CHECK-NEXT:    s_waitcnt lgkmcnt(0)382; CHECK-NEXT:    s_setpc_b64 s[30:31]383entry:384  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 16, i1 false)385  ret void386}387 388define void @memcpy_p0_p1_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {389; CHECK-LABEL: memcpy_p0_p1_sz31_align_16_16:390; CHECK:       ; %bb.0: ; %entry391; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)392; CHECK-NEXT:    s_clause 0x1393; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:15394; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off395; CHECK-NEXT:    s_waitcnt vmcnt(1)396; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:15397; CHECK-NEXT:    s_waitcnt vmcnt(0)398; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]399; CHECK-NEXT:    s_waitcnt lgkmcnt(0)400; CHECK-NEXT:    s_setpc_b64 s[30:31]401entry:402  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 31, i1 false)403  ret void404}405 406define void @memcpy_p0_p1_sz32_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {407; CHECK-LABEL: memcpy_p0_p1_sz32_align_16_16:408; CHECK:       ; %bb.0: ; %entry409; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)410; CHECK-NEXT:    s_clause 0x1411; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:16412; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off413; CHECK-NEXT:    s_waitcnt vmcnt(1)414; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:16415; CHECK-NEXT:    s_waitcnt vmcnt(0)416; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]417; CHECK-NEXT:    s_waitcnt lgkmcnt(0)418; CHECK-NEXT:    s_setpc_b64 s[30:31]419entry:420  tail call void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 32, i1 false)421  ret void422}423 424define void @memcpy_p0_p3_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {425; CHECK-LABEL: memcpy_p0_p3_sz16_align_1_1:426; CHECK:       ; %bb.0: ; %entry427; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)428; CHECK-NEXT:    ds_read_b128 v[2:5], v2429; CHECK-NEXT:    s_waitcnt lgkmcnt(0)430; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]431; CHECK-NEXT:    s_waitcnt lgkmcnt(0)432; CHECK-NEXT:    s_setpc_b64 s[30:31]433entry:434  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 16, i1 false)435  ret void436}437 438define void @memcpy_p0_p3_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {439; CHECK-LABEL: memcpy_p0_p3_sz31_align_1_1:440; CHECK:       ; %bb.0: ; %entry441; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)442; CHECK-NEXT:    ds_read_b64 v[7:8], v2 offset:23443; CHECK-NEXT:    ds_read_b128 v[3:6], v2444; CHECK-NEXT:    ds_read_b64 v[9:10], v2 offset:16445; CHECK-NEXT:    s_waitcnt lgkmcnt(2)446; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[7:8] offset:23447; CHECK-NEXT:    s_waitcnt lgkmcnt(2)448; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]449; CHECK-NEXT:    s_waitcnt lgkmcnt(2)450; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[9:10] offset:16451; CHECK-NEXT:    s_waitcnt lgkmcnt(0)452; CHECK-NEXT:    s_setpc_b64 s[30:31]453entry:454  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 31, i1 false)455  ret void456}457 458define void @memcpy_p0_p3_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {459; CHECK-LABEL: memcpy_p0_p3_sz32_align_1_1:460; CHECK:       ; %bb.0: ; %entry461; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)462; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:16463; CHECK-NEXT:    ds_read_b128 v[7:10], v2464; CHECK-NEXT:    s_waitcnt lgkmcnt(1)465; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16466; CHECK-NEXT:    s_waitcnt lgkmcnt(1)467; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]468; CHECK-NEXT:    s_waitcnt lgkmcnt(0)469; CHECK-NEXT:    s_setpc_b64 s[30:31]470entry:471  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 32, i1 false)472  ret void473}474 475define void @memcpy_p0_p3_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {476; CHECK-LABEL: memcpy_p0_p3_sz16_align_2_2:477; CHECK:       ; %bb.0: ; %entry478; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)479; CHECK-NEXT:    ds_read_b128 v[2:5], v2480; CHECK-NEXT:    s_waitcnt lgkmcnt(0)481; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]482; CHECK-NEXT:    s_waitcnt lgkmcnt(0)483; CHECK-NEXT:    s_setpc_b64 s[30:31]484entry:485  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 16, i1 false)486  ret void487}488 489define void @memcpy_p0_p3_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {490; CHECK-LABEL: memcpy_p0_p3_sz31_align_2_2:491; CHECK:       ; %bb.0: ; %entry492; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)493; CHECK-NEXT:    ds_read_b64 v[7:8], v2 offset:23494; CHECK-NEXT:    ds_read_b128 v[3:6], v2495; CHECK-NEXT:    ds_read_b64 v[9:10], v2 offset:16496; CHECK-NEXT:    s_waitcnt lgkmcnt(2)497; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[7:8] offset:23498; CHECK-NEXT:    s_waitcnt lgkmcnt(2)499; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]500; CHECK-NEXT:    s_waitcnt lgkmcnt(2)501; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[9:10] offset:16502; CHECK-NEXT:    s_waitcnt lgkmcnt(0)503; CHECK-NEXT:    s_setpc_b64 s[30:31]504entry:505  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 31, i1 false)506  ret void507}508 509define void @memcpy_p0_p3_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {510; CHECK-LABEL: memcpy_p0_p3_sz32_align_2_2:511; CHECK:       ; %bb.0: ; %entry512; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)513; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:16514; CHECK-NEXT:    ds_read_b128 v[7:10], v2515; CHECK-NEXT:    s_waitcnt lgkmcnt(1)516; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16517; CHECK-NEXT:    s_waitcnt lgkmcnt(1)518; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]519; CHECK-NEXT:    s_waitcnt lgkmcnt(0)520; CHECK-NEXT:    s_setpc_b64 s[30:31]521entry:522  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 32, i1 false)523  ret void524}525 526define void @memcpy_p0_p3_sz16_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {527; CHECK-LABEL: memcpy_p0_p3_sz16_align_8_8:528; CHECK:       ; %bb.0: ; %entry529; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)530; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:1531; CHECK-NEXT:    s_waitcnt lgkmcnt(0)532; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]533; CHECK-NEXT:    s_waitcnt lgkmcnt(0)534; CHECK-NEXT:    s_setpc_b64 s[30:31]535entry:536  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 16, i1 false)537  ret void538}539 540define void @memcpy_p0_p3_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {541; CHECK-LABEL: memcpy_p0_p3_sz31_align_8_8:542; CHECK:       ; %bb.0: ; %entry543; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)544; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:15545; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset1:1546; CHECK-NEXT:    s_waitcnt lgkmcnt(1)547; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:15548; CHECK-NEXT:    s_waitcnt lgkmcnt(1)549; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]550; CHECK-NEXT:    s_waitcnt lgkmcnt(0)551; CHECK-NEXT:    s_setpc_b64 s[30:31]552entry:553  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 31, i1 false)554  ret void555}556 557define void @memcpy_p0_p3_sz32_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {558; CHECK-LABEL: memcpy_p0_p3_sz32_align_8_8:559; CHECK:       ; %bb.0: ; %entry560; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)561; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset0:2 offset1:3562; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset1:1563; CHECK-NEXT:    s_waitcnt lgkmcnt(1)564; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16565; CHECK-NEXT:    s_waitcnt lgkmcnt(1)566; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]567; CHECK-NEXT:    s_waitcnt lgkmcnt(0)568; CHECK-NEXT:    s_setpc_b64 s[30:31]569entry:570  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 32, i1 false)571  ret void572}573 574define void @memcpy_p0_p3_sz16_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {575; CHECK-LABEL: memcpy_p0_p3_sz16_align_16_16:576; CHECK:       ; %bb.0: ; %entry577; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)578; CHECK-NEXT:    ds_read_b128 v[2:5], v2579; CHECK-NEXT:    s_waitcnt lgkmcnt(0)580; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]581; CHECK-NEXT:    s_waitcnt lgkmcnt(0)582; CHECK-NEXT:    s_setpc_b64 s[30:31]583entry:584  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 16, i1 false)585  ret void586}587 588define void @memcpy_p0_p3_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {589; CHECK-LABEL: memcpy_p0_p3_sz31_align_16_16:590; CHECK:       ; %bb.0: ; %entry591; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)592; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:15593; CHECK-NEXT:    ds_read_b128 v[7:10], v2594; CHECK-NEXT:    s_waitcnt lgkmcnt(1)595; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:15596; CHECK-NEXT:    s_waitcnt lgkmcnt(1)597; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]598; CHECK-NEXT:    s_waitcnt lgkmcnt(0)599; CHECK-NEXT:    s_setpc_b64 s[30:31]600entry:601  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 31, i1 false)602  ret void603}604 605define void @memcpy_p0_p3_sz32_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {606; CHECK-LABEL: memcpy_p0_p3_sz32_align_16_16:607; CHECK:       ; %bb.0: ; %entry608; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)609; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:16610; CHECK-NEXT:    ds_read_b128 v[7:10], v2611; CHECK-NEXT:    s_waitcnt lgkmcnt(1)612; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6] offset:16613; CHECK-NEXT:    s_waitcnt lgkmcnt(1)614; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]615; CHECK-NEXT:    s_waitcnt lgkmcnt(0)616; CHECK-NEXT:    s_setpc_b64 s[30:31]617entry:618  tail call void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 32, i1 false)619  ret void620}621 622define void @memcpy_p0_p4_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {623; CHECK-LABEL: memcpy_p0_p4_sz16_align_1_1:624; CHECK:       ; %bb.0: ; %entry625; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)626; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off627; CHECK-NEXT:    s_waitcnt vmcnt(0)628; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]629; CHECK-NEXT:    s_waitcnt lgkmcnt(0)630; CHECK-NEXT:    s_setpc_b64 s[30:31]631entry:632  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 16, i1 false)633  ret void634}635 636define void @memcpy_p0_p4_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {637; CHECK-LABEL: memcpy_p0_p4_sz31_align_1_1:638; CHECK:       ; %bb.0: ; %entry639; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)640; CHECK-NEXT:    s_clause 0x1641; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off642; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:8643; CHECK-NEXT:    s_waitcnt vmcnt(1)644; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[8:9]645; CHECK-NEXT:    s_waitcnt vmcnt(0)646; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:8647; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:23648; CHECK-NEXT:    s_waitcnt vmcnt(0)649; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:23650; CHECK-NEXT:    s_waitcnt lgkmcnt(0)651; CHECK-NEXT:    s_setpc_b64 s[30:31]652entry:653  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 31, i1 false)654  ret void655}656 657define void @memcpy_p0_p4_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {658; CHECK-LABEL: memcpy_p0_p4_sz32_align_1_1:659; CHECK:       ; %bb.0: ; %entry660; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)661; CHECK-NEXT:    s_clause 0x1662; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off663; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:16664; CHECK-NEXT:    s_waitcnt vmcnt(1)665; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]666; CHECK-NEXT:    s_waitcnt vmcnt(0)667; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11] offset:16668; CHECK-NEXT:    s_waitcnt lgkmcnt(0)669; CHECK-NEXT:    s_setpc_b64 s[30:31]670entry:671  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 32, i1 false)672  ret void673}674 675define void @memcpy_p0_p4_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {676; CHECK-LABEL: memcpy_p0_p4_sz16_align_2_2:677; CHECK:       ; %bb.0: ; %entry678; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)679; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off680; CHECK-NEXT:    s_waitcnt vmcnt(0)681; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]682; CHECK-NEXT:    s_waitcnt lgkmcnt(0)683; CHECK-NEXT:    s_setpc_b64 s[30:31]684entry:685  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 16, i1 false)686  ret void687}688 689define void @memcpy_p0_p4_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {690; CHECK-LABEL: memcpy_p0_p4_sz31_align_2_2:691; CHECK:       ; %bb.0: ; %entry692; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)693; CHECK-NEXT:    s_clause 0x1694; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off695; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:8696; CHECK-NEXT:    s_waitcnt vmcnt(1)697; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[8:9]698; CHECK-NEXT:    s_waitcnt vmcnt(0)699; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:8700; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:23701; CHECK-NEXT:    s_waitcnt vmcnt(0)702; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[2:3] offset:23703; CHECK-NEXT:    s_waitcnt lgkmcnt(0)704; CHECK-NEXT:    s_setpc_b64 s[30:31]705entry:706  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 31, i1 false)707  ret void708}709 710define void @memcpy_p0_p4_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {711; CHECK-LABEL: memcpy_p0_p4_sz32_align_2_2:712; CHECK:       ; %bb.0: ; %entry713; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)714; CHECK-NEXT:    s_clause 0x1715; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off716; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:16717; CHECK-NEXT:    s_waitcnt vmcnt(1)718; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]719; CHECK-NEXT:    s_waitcnt vmcnt(0)720; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[8:11] offset:16721; CHECK-NEXT:    s_waitcnt lgkmcnt(0)722; CHECK-NEXT:    s_setpc_b64 s[30:31]723entry:724  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 32, i1 false)725  ret void726}727 728define void @memcpy_p0_p4_sz16_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {729; CHECK-LABEL: memcpy_p0_p4_sz16_align_8_8:730; CHECK:       ; %bb.0: ; %entry731; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)732; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off733; CHECK-NEXT:    s_waitcnt vmcnt(0)734; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]735; CHECK-NEXT:    s_waitcnt lgkmcnt(0)736; CHECK-NEXT:    s_setpc_b64 s[30:31]737entry:738  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 16, i1 false)739  ret void740}741 742define void @memcpy_p0_p4_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {743; CHECK-LABEL: memcpy_p0_p4_sz31_align_8_8:744; CHECK:       ; %bb.0: ; %entry745; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)746; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off747; CHECK-NEXT:    s_waitcnt vmcnt(0)748; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]749; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:15750; CHECK-NEXT:    s_waitcnt vmcnt(0)751; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:15752; CHECK-NEXT:    s_waitcnt lgkmcnt(0)753; CHECK-NEXT:    s_setpc_b64 s[30:31]754entry:755  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 31, i1 false)756  ret void757}758 759define void @memcpy_p0_p4_sz32_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {760; CHECK-LABEL: memcpy_p0_p4_sz32_align_8_8:761; CHECK:       ; %bb.0: ; %entry762; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)763; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off764; CHECK-NEXT:    s_waitcnt vmcnt(0)765; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]766; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:16767; CHECK-NEXT:    s_waitcnt vmcnt(0)768; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:16769; CHECK-NEXT:    s_waitcnt lgkmcnt(0)770; CHECK-NEXT:    s_setpc_b64 s[30:31]771entry:772  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 32, i1 false)773  ret void774}775 776define void @memcpy_p0_p4_sz16_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {777; CHECK-LABEL: memcpy_p0_p4_sz16_align_16_16:778; CHECK:       ; %bb.0: ; %entry779; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)780; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off781; CHECK-NEXT:    s_waitcnt vmcnt(0)782; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]783; CHECK-NEXT:    s_waitcnt lgkmcnt(0)784; CHECK-NEXT:    s_setpc_b64 s[30:31]785entry:786  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 16, i1 false)787  ret void788}789 790define void @memcpy_p0_p4_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {791; CHECK-LABEL: memcpy_p0_p4_sz31_align_16_16:792; CHECK:       ; %bb.0: ; %entry793; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)794; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off795; CHECK-NEXT:    s_waitcnt vmcnt(0)796; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]797; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:15798; CHECK-NEXT:    s_waitcnt vmcnt(0)799; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:15800; CHECK-NEXT:    s_waitcnt lgkmcnt(0)801; CHECK-NEXT:    s_setpc_b64 s[30:31]802entry:803  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 31, i1 false)804  ret void805}806 807define void @memcpy_p0_p4_sz32_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {808; CHECK-LABEL: memcpy_p0_p4_sz32_align_16_16:809; CHECK:       ; %bb.0: ; %entry810; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)811; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off812; CHECK-NEXT:    s_waitcnt vmcnt(0)813; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]814; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:16815; CHECK-NEXT:    s_waitcnt vmcnt(0)816; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[2:5] offset:16817; CHECK-NEXT:    s_waitcnt lgkmcnt(0)818; CHECK-NEXT:    s_setpc_b64 s[30:31]819entry:820  tail call void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 32, i1 false)821  ret void822}823 824define void @memcpy_p0_p5_sz16_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {825; CHECK-LABEL: memcpy_p0_p5_sz16_align_1_1:826; CHECK:       ; %bb.0: ; %entry827; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)828; CHECK-NEXT:    s_clause 0x3829; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen830; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4831; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8832; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12833; CHECK-NEXT:    s_waitcnt vmcnt(0)834; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]835; CHECK-NEXT:    s_waitcnt lgkmcnt(0)836; CHECK-NEXT:    s_setpc_b64 s[30:31]837entry:838  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)839  ret void840}841 842define void @memcpy_p0_p5_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {843; CHECK-LABEL: memcpy_p0_p5_sz31_align_1_1:844; CHECK:       ; %bb.0: ; %entry845; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)846; CHECK-NEXT:    s_clause 0x7847; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen848; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4849; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8850; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12851; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:23852; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:27853; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:16854; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:20855; CHECK-NEXT:    s_waitcnt vmcnt(2)856; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[7:8] offset:23857; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]858; CHECK-NEXT:    s_waitcnt vmcnt(0)859; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[9:10] offset:16860; CHECK-NEXT:    s_waitcnt lgkmcnt(0)861; CHECK-NEXT:    s_setpc_b64 s[30:31]862entry:863  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)864  ret void865}866 867define void @memcpy_p0_p5_sz32_align_1_1(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {868; CHECK-LABEL: memcpy_p0_p5_sz32_align_1_1:869; CHECK:       ; %bb.0: ; %entry870; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)871; CHECK-NEXT:    s_clause 0x7872; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen873; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4874; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8875; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16876; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20877; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24878; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28879; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12880; CHECK-NEXT:    s_waitcnt vmcnt(1)881; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:16882; CHECK-NEXT:    s_waitcnt vmcnt(0)883; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]884; CHECK-NEXT:    s_waitcnt lgkmcnt(0)885; CHECK-NEXT:    s_setpc_b64 s[30:31]886entry:887  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)888  ret void889}890 891define void @memcpy_p0_p5_sz16_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {892; CHECK-LABEL: memcpy_p0_p5_sz16_align_2_2:893; CHECK:       ; %bb.0: ; %entry894; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)895; CHECK-NEXT:    s_clause 0x3896; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen897; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4898; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8899; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12900; CHECK-NEXT:    s_waitcnt vmcnt(0)901; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]902; CHECK-NEXT:    s_waitcnt lgkmcnt(0)903; CHECK-NEXT:    s_setpc_b64 s[30:31]904entry:905  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)906  ret void907}908 909define void @memcpy_p0_p5_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {910; CHECK-LABEL: memcpy_p0_p5_sz31_align_2_2:911; CHECK:       ; %bb.0: ; %entry912; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)913; CHECK-NEXT:    s_clause 0x7914; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen915; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4916; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8917; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12918; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:23919; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:27920; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:16921; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:20922; CHECK-NEXT:    s_waitcnt vmcnt(2)923; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[7:8] offset:23924; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]925; CHECK-NEXT:    s_waitcnt vmcnt(0)926; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[9:10] offset:16927; CHECK-NEXT:    s_waitcnt lgkmcnt(0)928; CHECK-NEXT:    s_setpc_b64 s[30:31]929entry:930  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)931  ret void932}933 934define void @memcpy_p0_p5_sz32_align_2_2(ptr addrspace(0) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {935; CHECK-LABEL: memcpy_p0_p5_sz32_align_2_2:936; CHECK:       ; %bb.0: ; %entry937; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)938; CHECK-NEXT:    s_clause 0x7939; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen940; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4941; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8942; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:16943; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:20944; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:24945; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:28946; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12947; CHECK-NEXT:    s_waitcnt vmcnt(1)948; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:16949; CHECK-NEXT:    s_waitcnt vmcnt(0)950; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]951; CHECK-NEXT:    s_waitcnt lgkmcnt(0)952; CHECK-NEXT:    s_setpc_b64 s[30:31]953entry:954  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)955  ret void956}957 958define void @memcpy_p0_p5_sz16_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {959; CHECK-LABEL: memcpy_p0_p5_sz16_align_8_8:960; CHECK:       ; %bb.0: ; %entry961; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)962; CHECK-NEXT:    s_clause 0x3963; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen964; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4965; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8966; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12967; CHECK-NEXT:    s_waitcnt vmcnt(0)968; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]969; CHECK-NEXT:    s_waitcnt lgkmcnt(0)970; CHECK-NEXT:    s_setpc_b64 s[30:31]971entry:972  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 16, i1 false)973  ret void974}975 976define void @memcpy_p0_p5_sz31_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {977; CHECK-LABEL: memcpy_p0_p5_sz31_align_8_8:978; CHECK:       ; %bb.0: ; %entry979; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)980; CHECK-NEXT:    s_clause 0x7981; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen982; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:4983; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:8984; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:12985; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:15986; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:19987; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:23988; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:27989; CHECK-NEXT:    s_waitcnt vmcnt(4)990; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]991; CHECK-NEXT:    s_waitcnt vmcnt(0)992; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:15993; CHECK-NEXT:    s_waitcnt lgkmcnt(0)994; CHECK-NEXT:    s_setpc_b64 s[30:31]995entry:996  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 31, i1 false)997  ret void998}999 1000define void @memcpy_p0_p5_sz32_align_8_8(ptr addrspace(0) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {1001; CHECK-LABEL: memcpy_p0_p5_sz32_align_8_8:1002; CHECK:       ; %bb.0: ; %entry1003; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1004; CHECK-NEXT:    s_clause 0x71005; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1006; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41007; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81008; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121009; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:161010; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:201011; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:241012; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:281013; CHECK-NEXT:    s_waitcnt vmcnt(4)1014; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]1015; CHECK-NEXT:    s_waitcnt vmcnt(0)1016; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:161017; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1018; CHECK-NEXT:    s_setpc_b64 s[30:31]1019entry:1020  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 32, i1 false)1021  ret void1022}1023 1024define void @memcpy_p0_p5_sz16_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {1025; CHECK-LABEL: memcpy_p0_p5_sz16_align_16_16:1026; CHECK:       ; %bb.0: ; %entry1027; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1028; CHECK-NEXT:    s_clause 0x31029; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1030; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41031; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81032; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121033; CHECK-NEXT:    s_waitcnt vmcnt(0)1034; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]1035; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1036; CHECK-NEXT:    s_setpc_b64 s[30:31]1037entry:1038  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 16, i1 false)1039  ret void1040}1041 1042define void @memcpy_p0_p5_sz31_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {1043; CHECK-LABEL: memcpy_p0_p5_sz31_align_16_16:1044; CHECK:       ; %bb.0: ; %entry1045; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1046; CHECK-NEXT:    s_clause 0x71047; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1048; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41049; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81050; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121051; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:151052; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:191053; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:231054; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:271055; CHECK-NEXT:    s_waitcnt vmcnt(4)1056; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]1057; CHECK-NEXT:    s_waitcnt vmcnt(0)1058; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:151059; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1060; CHECK-NEXT:    s_setpc_b64 s[30:31]1061entry:1062  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 31, i1 false)1063  ret void1064}1065 1066define void @memcpy_p0_p5_sz32_align_16_16(ptr addrspace(0) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {1067; CHECK-LABEL: memcpy_p0_p5_sz32_align_16_16:1068; CHECK:       ; %bb.0: ; %entry1069; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1070; CHECK-NEXT:    s_clause 0x71071; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1072; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41073; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81074; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121075; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:161076; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:201077; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:241078; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:281079; CHECK-NEXT:    s_waitcnt vmcnt(4)1080; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]1081; CHECK-NEXT:    s_waitcnt vmcnt(0)1082; CHECK-NEXT:    flat_store_dwordx4 v[0:1], v[7:10] offset:161083; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1084; CHECK-NEXT:    s_setpc_b64 s[30:31]1085entry:1086  tail call void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 32, i1 false)1087  ret void1088}1089 1090define void @memcpy_p1_p0_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {1091; CHECK-LABEL: memcpy_p1_p0_sz16_align_1_1:1092; CHECK:       ; %bb.0: ; %entry1093; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1094; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]1095; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1096; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1097; CHECK-NEXT:    s_setpc_b64 s[30:31]1098entry:1099  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 16, i1 false)1100  ret void1101}1102 1103define void @memcpy_p1_p0_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {1104; CHECK-LABEL: memcpy_p1_p0_sz31_align_1_1:1105; CHECK:       ; %bb.0: ; %entry1106; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1107; CHECK-NEXT:    s_clause 0x21108; CHECK-NEXT:    flat_load_dwordx2 v[8:9], v[2:3] offset:231109; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3]1110; CHECK-NEXT:    flat_load_dwordx2 v[2:3], v[2:3] offset:161111; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)1112; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:231113; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1114; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1115; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1116; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off offset:161117; CHECK-NEXT:    s_setpc_b64 s[30:31]1118entry:1119  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 31, i1 false)1120  ret void1121}1122 1123define void @memcpy_p1_p0_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {1124; CHECK-LABEL: memcpy_p1_p0_sz32_align_1_1:1125; CHECK:       ; %bb.0: ; %entry1126; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1127; CHECK-NEXT:    s_clause 0x11128; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:161129; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]1130; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1131; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161132; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1133; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1134; CHECK-NEXT:    s_setpc_b64 s[30:31]1135entry:1136  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)1137  ret void1138}1139 1140define void @memcpy_p1_p0_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {1141; CHECK-LABEL: memcpy_p1_p0_sz16_align_2_2:1142; CHECK:       ; %bb.0: ; %entry1143; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1144; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]1145; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1146; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1147; CHECK-NEXT:    s_setpc_b64 s[30:31]1148entry:1149  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 16, i1 false)1150  ret void1151}1152 1153define void @memcpy_p1_p0_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {1154; CHECK-LABEL: memcpy_p1_p0_sz31_align_2_2:1155; CHECK:       ; %bb.0: ; %entry1156; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1157; CHECK-NEXT:    s_clause 0x21158; CHECK-NEXT:    flat_load_dwordx2 v[8:9], v[2:3] offset:231159; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3]1160; CHECK-NEXT:    flat_load_dwordx2 v[2:3], v[2:3] offset:161161; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)1162; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:231163; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1164; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1165; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1166; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off offset:161167; CHECK-NEXT:    s_setpc_b64 s[30:31]1168entry:1169  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 31, i1 false)1170  ret void1171}1172 1173define void @memcpy_p1_p0_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {1174; CHECK-LABEL: memcpy_p1_p0_sz32_align_2_2:1175; CHECK:       ; %bb.0: ; %entry1176; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1177; CHECK-NEXT:    s_clause 0x11178; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:161179; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]1180; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1181; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161182; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1183; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1184; CHECK-NEXT:    s_setpc_b64 s[30:31]1185entry:1186  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)1187  ret void1188}1189 1190define void @memcpy_p1_p0_sz16_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {1191; CHECK-LABEL: memcpy_p1_p0_sz16_align_8_8:1192; CHECK:       ; %bb.0: ; %entry1193; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1194; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]1195; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1196; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1197; CHECK-NEXT:    s_setpc_b64 s[30:31]1198entry:1199  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 16, i1 false)1200  ret void1201}1202 1203define void @memcpy_p1_p0_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {1204; CHECK-LABEL: memcpy_p1_p0_sz31_align_8_8:1205; CHECK:       ; %bb.0: ; %entry1206; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1207; CHECK-NEXT:    s_clause 0x11208; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:151209; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]1210; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1211; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:151212; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1213; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1214; CHECK-NEXT:    s_setpc_b64 s[30:31]1215entry:1216  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 31, i1 false)1217  ret void1218}1219 1220define void @memcpy_p1_p0_sz32_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {1221; CHECK-LABEL: memcpy_p1_p0_sz32_align_8_8:1222; CHECK:       ; %bb.0: ; %entry1223; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1224; CHECK-NEXT:    s_clause 0x11225; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:161226; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]1227; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1228; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161229; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1230; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1231; CHECK-NEXT:    s_setpc_b64 s[30:31]1232entry:1233  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 32, i1 false)1234  ret void1235}1236 1237define void @memcpy_p1_p0_sz16_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {1238; CHECK-LABEL: memcpy_p1_p0_sz16_align_16_16:1239; CHECK:       ; %bb.0: ; %entry1240; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1241; CHECK-NEXT:    flat_load_dwordx4 v[2:5], v[2:3]1242; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1243; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1244; CHECK-NEXT:    s_setpc_b64 s[30:31]1245entry:1246  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 16, i1 false)1247  ret void1248}1249 1250define void @memcpy_p1_p0_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {1251; CHECK-LABEL: memcpy_p1_p0_sz31_align_16_16:1252; CHECK:       ; %bb.0: ; %entry1253; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1254; CHECK-NEXT:    s_clause 0x11255; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:151256; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]1257; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1258; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:151259; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1260; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1261; CHECK-NEXT:    s_setpc_b64 s[30:31]1262entry:1263  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 31, i1 false)1264  ret void1265}1266 1267define void @memcpy_p1_p0_sz32_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {1268; CHECK-LABEL: memcpy_p1_p0_sz32_align_16_16:1269; CHECK:       ; %bb.0: ; %entry1270; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1271; CHECK-NEXT:    s_clause 0x11272; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[2:3] offset:161273; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[2:3]1274; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)1275; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161276; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1277; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1278; CHECK-NEXT:    s_setpc_b64 s[30:31]1279entry:1280  tail call void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 32, i1 false)1281  ret void1282}1283 1284define void @memcpy_p1_p1_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {1285; CHECK-LABEL: memcpy_p1_p1_sz16_align_1_1:1286; CHECK:       ; %bb.0: ; %entry1287; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1288; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1289; CHECK-NEXT:    s_waitcnt vmcnt(0)1290; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1291; CHECK-NEXT:    s_setpc_b64 s[30:31]1292entry:1293  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 16, i1 false)1294  ret void1295}1296 1297define void @memcpy_p1_p1_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {1298; CHECK-LABEL: memcpy_p1_p1_sz31_align_1_1:1299; CHECK:       ; %bb.0: ; %entry1300; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1301; CHECK-NEXT:    s_clause 0x21302; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off offset:231303; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1304; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:161305; CHECK-NEXT:    s_waitcnt vmcnt(2)1306; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:231307; CHECK-NEXT:    s_waitcnt vmcnt(1)1308; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1309; CHECK-NEXT:    s_waitcnt vmcnt(0)1310; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off offset:161311; CHECK-NEXT:    s_setpc_b64 s[30:31]1312entry:1313  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 31, i1 false)1314  ret void1315}1316 1317define void @memcpy_p1_p1_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {1318; CHECK-LABEL: memcpy_p1_p1_sz32_align_1_1:1319; CHECK:       ; %bb.0: ; %entry1320; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1321; CHECK-NEXT:    s_clause 0x11322; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:161323; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off1324; CHECK-NEXT:    s_waitcnt vmcnt(1)1325; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161326; CHECK-NEXT:    s_waitcnt vmcnt(0)1327; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1328; CHECK-NEXT:    s_setpc_b64 s[30:31]1329entry:1330  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 32, i1 false)1331  ret void1332}1333 1334define void @memcpy_p1_p1_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {1335; CHECK-LABEL: memcpy_p1_p1_sz16_align_2_2:1336; CHECK:       ; %bb.0: ; %entry1337; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1338; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1339; CHECK-NEXT:    s_waitcnt vmcnt(0)1340; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1341; CHECK-NEXT:    s_setpc_b64 s[30:31]1342entry:1343  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 16, i1 false)1344  ret void1345}1346 1347define void @memcpy_p1_p1_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {1348; CHECK-LABEL: memcpy_p1_p1_sz31_align_2_2:1349; CHECK:       ; %bb.0: ; %entry1350; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1351; CHECK-NEXT:    s_clause 0x21352; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off offset:231353; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1354; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:161355; CHECK-NEXT:    s_waitcnt vmcnt(2)1356; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off offset:231357; CHECK-NEXT:    s_waitcnt vmcnt(1)1358; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1359; CHECK-NEXT:    s_waitcnt vmcnt(0)1360; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off offset:161361; CHECK-NEXT:    s_setpc_b64 s[30:31]1362entry:1363  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 31, i1 false)1364  ret void1365}1366 1367define void @memcpy_p1_p1_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {1368; CHECK-LABEL: memcpy_p1_p1_sz32_align_2_2:1369; CHECK:       ; %bb.0: ; %entry1370; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1371; CHECK-NEXT:    s_clause 0x11372; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:161373; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off1374; CHECK-NEXT:    s_waitcnt vmcnt(1)1375; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161376; CHECK-NEXT:    s_waitcnt vmcnt(0)1377; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1378; CHECK-NEXT:    s_setpc_b64 s[30:31]1379entry:1380  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 32, i1 false)1381  ret void1382}1383 1384define void @memcpy_p1_p1_sz16_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {1385; CHECK-LABEL: memcpy_p1_p1_sz16_align_8_8:1386; CHECK:       ; %bb.0: ; %entry1387; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1388; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1389; CHECK-NEXT:    s_waitcnt vmcnt(0)1390; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1391; CHECK-NEXT:    s_setpc_b64 s[30:31]1392entry:1393  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 16, i1 false)1394  ret void1395}1396 1397define void @memcpy_p1_p1_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {1398; CHECK-LABEL: memcpy_p1_p1_sz31_align_8_8:1399; CHECK:       ; %bb.0: ; %entry1400; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1401; CHECK-NEXT:    s_clause 0x11402; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:151403; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off1404; CHECK-NEXT:    s_waitcnt vmcnt(1)1405; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:151406; CHECK-NEXT:    s_waitcnt vmcnt(0)1407; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1408; CHECK-NEXT:    s_setpc_b64 s[30:31]1409entry:1410  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 31, i1 false)1411  ret void1412}1413 1414define void @memcpy_p1_p1_sz32_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {1415; CHECK-LABEL: memcpy_p1_p1_sz32_align_8_8:1416; CHECK:       ; %bb.0: ; %entry1417; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1418; CHECK-NEXT:    s_clause 0x11419; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:161420; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off1421; CHECK-NEXT:    s_waitcnt vmcnt(1)1422; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161423; CHECK-NEXT:    s_waitcnt vmcnt(0)1424; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1425; CHECK-NEXT:    s_setpc_b64 s[30:31]1426entry:1427  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 32, i1 false)1428  ret void1429}1430 1431define void @memcpy_p1_p1_sz16_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {1432; CHECK-LABEL: memcpy_p1_p1_sz16_align_16_16:1433; CHECK:       ; %bb.0: ; %entry1434; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1435; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1436; CHECK-NEXT:    s_waitcnt vmcnt(0)1437; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1438; CHECK-NEXT:    s_setpc_b64 s[30:31]1439entry:1440  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 16, i1 false)1441  ret void1442}1443 1444define void @memcpy_p1_p1_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {1445; CHECK-LABEL: memcpy_p1_p1_sz31_align_16_16:1446; CHECK:       ; %bb.0: ; %entry1447; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1448; CHECK-NEXT:    s_clause 0x11449; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:151450; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off1451; CHECK-NEXT:    s_waitcnt vmcnt(1)1452; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:151453; CHECK-NEXT:    s_waitcnt vmcnt(0)1454; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1455; CHECK-NEXT:    s_setpc_b64 s[30:31]1456entry:1457  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 31, i1 false)1458  ret void1459}1460 1461define void @memcpy_p1_p1_sz32_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {1462; CHECK-LABEL: memcpy_p1_p1_sz32_align_16_16:1463; CHECK:       ; %bb.0: ; %entry1464; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1465; CHECK-NEXT:    s_clause 0x11466; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:161467; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off1468; CHECK-NEXT:    s_waitcnt vmcnt(1)1469; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:161470; CHECK-NEXT:    s_waitcnt vmcnt(0)1471; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off1472; CHECK-NEXT:    s_setpc_b64 s[30:31]1473entry:1474  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 32, i1 false)1475  ret void1476}1477 1478define void @memcpy_p1_p3_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {1479; CHECK-LABEL: memcpy_p1_p3_sz16_align_1_1:1480; CHECK:       ; %bb.0: ; %entry1481; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1482; CHECK-NEXT:    ds_read_b128 v[2:5], v21483; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1484; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1485; CHECK-NEXT:    s_setpc_b64 s[30:31]1486entry:1487  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 16, i1 false)1488  ret void1489}1490 1491define void @memcpy_p1_p3_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {1492; CHECK-LABEL: memcpy_p1_p3_sz31_align_1_1:1493; CHECK:       ; %bb.0: ; %entry1494; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1495; CHECK-NEXT:    ds_read_b64 v[7:8], v21496; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:81497; CHECK-NEXT:    ds_read_b64 v[9:10], v2 offset:231498; CHECK-NEXT:    s_waitcnt lgkmcnt(2)1499; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[7:8], off1500; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1501; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off offset:81502; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1503; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[9:10], off offset:231504; CHECK-NEXT:    s_setpc_b64 s[30:31]1505entry:1506  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 31, i1 false)1507  ret void1508}1509 1510define void @memcpy_p1_p3_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {1511; CHECK-LABEL: memcpy_p1_p3_sz32_align_1_1:1512; CHECK:       ; %bb.0: ; %entry1513; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1514; CHECK-NEXT:    ds_read_b128 v[3:6], v21515; CHECK-NEXT:    ds_read_b128 v[7:10], v2 offset:161516; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1517; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1518; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1519; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:161520; CHECK-NEXT:    s_setpc_b64 s[30:31]1521entry:1522  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 32, i1 false)1523  ret void1524}1525 1526define void @memcpy_p1_p3_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {1527; CHECK-LABEL: memcpy_p1_p3_sz16_align_2_2:1528; CHECK:       ; %bb.0: ; %entry1529; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1530; CHECK-NEXT:    ds_read_b128 v[2:5], v21531; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1532; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1533; CHECK-NEXT:    s_setpc_b64 s[30:31]1534entry:1535  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 16, i1 false)1536  ret void1537}1538 1539define void @memcpy_p1_p3_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {1540; CHECK-LABEL: memcpy_p1_p3_sz31_align_2_2:1541; CHECK:       ; %bb.0: ; %entry1542; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1543; CHECK-NEXT:    ds_read_b64 v[7:8], v21544; CHECK-NEXT:    ds_read_b128 v[3:6], v2 offset:81545; CHECK-NEXT:    ds_read_b64 v[9:10], v2 offset:231546; CHECK-NEXT:    s_waitcnt lgkmcnt(2)1547; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[7:8], off1548; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1549; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off offset:81550; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1551; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[9:10], off offset:231552; CHECK-NEXT:    s_setpc_b64 s[30:31]1553entry:1554  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 31, i1 false)1555  ret void1556}1557 1558define void @memcpy_p1_p3_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {1559; CHECK-LABEL: memcpy_p1_p3_sz32_align_2_2:1560; CHECK:       ; %bb.0: ; %entry1561; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1562; CHECK-NEXT:    ds_read_b128 v[3:6], v21563; CHECK-NEXT:    ds_read_b128 v[7:10], v2 offset:161564; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1565; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1566; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1567; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:161568; CHECK-NEXT:    s_setpc_b64 s[30:31]1569entry:1570  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 32, i1 false)1571  ret void1572}1573 1574define void @memcpy_p1_p3_sz16_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {1575; CHECK-LABEL: memcpy_p1_p3_sz16_align_8_8:1576; CHECK:       ; %bb.0: ; %entry1577; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1578; CHECK-NEXT:    ds_read2_b64 v[2:5], v2 offset1:11579; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1580; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1581; CHECK-NEXT:    s_setpc_b64 s[30:31]1582entry:1583  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 16, i1 false)1584  ret void1585}1586 1587define void @memcpy_p1_p3_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {1588; CHECK-LABEL: memcpy_p1_p3_sz31_align_8_8:1589; CHECK:       ; %bb.0: ; %entry1590; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1591; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset1:11592; CHECK-NEXT:    ds_read_b128 v[7:10], v2 offset:151593; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1594; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1595; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1596; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:151597; CHECK-NEXT:    s_setpc_b64 s[30:31]1598entry:1599  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 31, i1 false)1600  ret void1601}1602 1603define void @memcpy_p1_p3_sz32_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {1604; CHECK-LABEL: memcpy_p1_p3_sz32_align_8_8:1605; CHECK:       ; %bb.0: ; %entry1606; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1607; CHECK-NEXT:    ds_read2_b64 v[3:6], v2 offset1:11608; CHECK-NEXT:    ds_read2_b64 v[7:10], v2 offset0:2 offset1:31609; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1610; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1611; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1612; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:161613; CHECK-NEXT:    s_setpc_b64 s[30:31]1614entry:1615  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 32, i1 false)1616  ret void1617}1618 1619define void @memcpy_p1_p3_sz16_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {1620; CHECK-LABEL: memcpy_p1_p3_sz16_align_16_16:1621; CHECK:       ; %bb.0: ; %entry1622; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1623; CHECK-NEXT:    ds_read_b128 v[2:5], v21624; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1625; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1626; CHECK-NEXT:    s_setpc_b64 s[30:31]1627entry:1628  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 16, i1 false)1629  ret void1630}1631 1632define void @memcpy_p1_p3_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {1633; CHECK-LABEL: memcpy_p1_p3_sz31_align_16_16:1634; CHECK:       ; %bb.0: ; %entry1635; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1636; CHECK-NEXT:    ds_read_b128 v[3:6], v21637; CHECK-NEXT:    ds_read_b128 v[7:10], v2 offset:151638; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1639; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1640; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1641; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:151642; CHECK-NEXT:    s_setpc_b64 s[30:31]1643entry:1644  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 31, i1 false)1645  ret void1646}1647 1648define void @memcpy_p1_p3_sz32_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {1649; CHECK-LABEL: memcpy_p1_p3_sz32_align_16_16:1650; CHECK:       ; %bb.0: ; %entry1651; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1652; CHECK-NEXT:    ds_read_b128 v[3:6], v21653; CHECK-NEXT:    ds_read_b128 v[7:10], v2 offset:161654; CHECK-NEXT:    s_waitcnt lgkmcnt(1)1655; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1656; CHECK-NEXT:    s_waitcnt lgkmcnt(0)1657; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:161658; CHECK-NEXT:    s_setpc_b64 s[30:31]1659entry:1660  tail call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 32, i1 false)1661  ret void1662}1663 1664define void @memcpy_p1_p4_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {1665; CHECK-LABEL: memcpy_p1_p4_sz16_align_1_1:1666; CHECK:       ; %bb.0: ; %entry1667; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1668; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1669; CHECK-NEXT:    s_waitcnt vmcnt(0)1670; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1671; CHECK-NEXT:    s_setpc_b64 s[30:31]1672entry:1673  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 16, i1 false)1674  ret void1675}1676 1677define void @memcpy_p1_p4_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {1678; CHECK-LABEL: memcpy_p1_p4_sz31_align_1_1:1679; CHECK:       ; %bb.0: ; %entry1680; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1681; CHECK-NEXT:    s_clause 0x11682; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off1683; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:81684; CHECK-NEXT:    s_waitcnt vmcnt(1)1685; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off1686; CHECK-NEXT:    s_waitcnt vmcnt(0)1687; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:81688; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:231689; CHECK-NEXT:    s_waitcnt vmcnt(0)1690; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off offset:231691; CHECK-NEXT:    s_setpc_b64 s[30:31]1692entry:1693  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 31, i1 false)1694  ret void1695}1696 1697define void @memcpy_p1_p4_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {1698; CHECK-LABEL: memcpy_p1_p4_sz32_align_1_1:1699; CHECK:       ; %bb.0: ; %entry1700; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1701; CHECK-NEXT:    s_clause 0x11702; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1703; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:161704; CHECK-NEXT:    s_waitcnt vmcnt(1)1705; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1706; CHECK-NEXT:    s_waitcnt vmcnt(0)1707; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off offset:161708; CHECK-NEXT:    s_setpc_b64 s[30:31]1709entry:1710  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 32, i1 false)1711  ret void1712}1713 1714define void @memcpy_p1_p4_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {1715; CHECK-LABEL: memcpy_p1_p4_sz16_align_2_2:1716; CHECK:       ; %bb.0: ; %entry1717; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1718; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1719; CHECK-NEXT:    s_waitcnt vmcnt(0)1720; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1721; CHECK-NEXT:    s_setpc_b64 s[30:31]1722entry:1723  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 16, i1 false)1724  ret void1725}1726 1727define void @memcpy_p1_p4_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {1728; CHECK-LABEL: memcpy_p1_p4_sz31_align_2_2:1729; CHECK:       ; %bb.0: ; %entry1730; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1731; CHECK-NEXT:    s_clause 0x11732; CHECK-NEXT:    global_load_dwordx2 v[8:9], v[2:3], off1733; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off offset:81734; CHECK-NEXT:    s_waitcnt vmcnt(1)1735; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[8:9], off1736; CHECK-NEXT:    s_waitcnt vmcnt(0)1737; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off offset:81738; CHECK-NEXT:    global_load_dwordx2 v[2:3], v[2:3], off offset:231739; CHECK-NEXT:    s_waitcnt vmcnt(0)1740; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off offset:231741; CHECK-NEXT:    s_setpc_b64 s[30:31]1742entry:1743  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 31, i1 false)1744  ret void1745}1746 1747define void @memcpy_p1_p4_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {1748; CHECK-LABEL: memcpy_p1_p4_sz32_align_2_2:1749; CHECK:       ; %bb.0: ; %entry1750; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1751; CHECK-NEXT:    s_clause 0x11752; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1753; CHECK-NEXT:    global_load_dwordx4 v[8:11], v[2:3], off offset:161754; CHECK-NEXT:    s_waitcnt vmcnt(1)1755; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1756; CHECK-NEXT:    s_waitcnt vmcnt(0)1757; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off offset:161758; CHECK-NEXT:    s_setpc_b64 s[30:31]1759entry:1760  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 32, i1 false)1761  ret void1762}1763 1764define void @memcpy_p1_p4_sz16_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {1765; CHECK-LABEL: memcpy_p1_p4_sz16_align_8_8:1766; CHECK:       ; %bb.0: ; %entry1767; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1768; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1769; CHECK-NEXT:    s_waitcnt vmcnt(0)1770; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1771; CHECK-NEXT:    s_setpc_b64 s[30:31]1772entry:1773  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 16, i1 false)1774  ret void1775}1776 1777define void @memcpy_p1_p4_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {1778; CHECK-LABEL: memcpy_p1_p4_sz31_align_8_8:1779; CHECK:       ; %bb.0: ; %entry1780; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1781; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1782; CHECK-NEXT:    s_waitcnt vmcnt(0)1783; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1784; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:151785; CHECK-NEXT:    s_waitcnt vmcnt(0)1786; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:151787; CHECK-NEXT:    s_setpc_b64 s[30:31]1788entry:1789  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 31, i1 false)1790  ret void1791}1792 1793define void @memcpy_p1_p4_sz32_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {1794; CHECK-LABEL: memcpy_p1_p4_sz32_align_8_8:1795; CHECK:       ; %bb.0: ; %entry1796; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1797; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1798; CHECK-NEXT:    s_waitcnt vmcnt(0)1799; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1800; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:161801; CHECK-NEXT:    s_waitcnt vmcnt(0)1802; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:161803; CHECK-NEXT:    s_setpc_b64 s[30:31]1804entry:1805  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 32, i1 false)1806  ret void1807}1808 1809define void @memcpy_p1_p4_sz16_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {1810; CHECK-LABEL: memcpy_p1_p4_sz16_align_16_16:1811; CHECK:       ; %bb.0: ; %entry1812; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1813; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off1814; CHECK-NEXT:    s_waitcnt vmcnt(0)1815; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off1816; CHECK-NEXT:    s_setpc_b64 s[30:31]1817entry:1818  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 16, i1 false)1819  ret void1820}1821 1822define void @memcpy_p1_p4_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {1823; CHECK-LABEL: memcpy_p1_p4_sz31_align_16_16:1824; CHECK:       ; %bb.0: ; %entry1825; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1826; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1827; CHECK-NEXT:    s_waitcnt vmcnt(0)1828; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1829; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:151830; CHECK-NEXT:    s_waitcnt vmcnt(0)1831; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:151832; CHECK-NEXT:    s_setpc_b64 s[30:31]1833entry:1834  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 31, i1 false)1835  ret void1836}1837 1838define void @memcpy_p1_p4_sz32_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {1839; CHECK-LABEL: memcpy_p1_p4_sz32_align_16_16:1840; CHECK:       ; %bb.0: ; %entry1841; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1842; CHECK-NEXT:    global_load_dwordx4 v[4:7], v[2:3], off1843; CHECK-NEXT:    s_waitcnt vmcnt(0)1844; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off1845; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off offset:161846; CHECK-NEXT:    s_waitcnt vmcnt(0)1847; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:161848; CHECK-NEXT:    s_setpc_b64 s[30:31]1849entry:1850  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 32, i1 false)1851  ret void1852}1853 1854define void @memcpy_p1_p5_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {1855; CHECK-LABEL: memcpy_p1_p5_sz16_align_1_1:1856; CHECK:       ; %bb.0: ; %entry1857; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1858; CHECK-NEXT:    s_clause 0x31859; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1860; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41861; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81862; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121863; CHECK-NEXT:    s_waitcnt vmcnt(0)1864; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1865; CHECK-NEXT:    s_setpc_b64 s[30:31]1866entry:1867  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)1868  ret void1869}1870 1871define void @memcpy_p1_p5_sz31_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {1872; CHECK-LABEL: memcpy_p1_p5_sz31_align_1_1:1873; CHECK:       ; %bb.0: ; %entry1874; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1875; CHECK-NEXT:    s_clause 0x71876; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen1877; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:41878; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:81879; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:121880; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:161881; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:201882; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:231883; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:271884; CHECK-NEXT:    s_waitcnt vmcnt(6)1885; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[7:8], off1886; CHECK-NEXT:    s_waitcnt vmcnt(2)1887; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off offset:81888; CHECK-NEXT:    s_waitcnt vmcnt(0)1889; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[9:10], off offset:231890; CHECK-NEXT:    s_setpc_b64 s[30:31]1891entry:1892  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)1893  ret void1894}1895 1896define void @memcpy_p1_p5_sz32_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {1897; CHECK-LABEL: memcpy_p1_p5_sz32_align_1_1:1898; CHECK:       ; %bb.0: ; %entry1899; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1900; CHECK-NEXT:    s_clause 0x71901; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1902; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41903; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81904; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121905; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:161906; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:201907; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:241908; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:281909; CHECK-NEXT:    s_waitcnt vmcnt(4)1910; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1911; CHECK-NEXT:    s_waitcnt vmcnt(0)1912; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:161913; CHECK-NEXT:    s_setpc_b64 s[30:31]1914entry:1915  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)1916  ret void1917}1918 1919define void @memcpy_p1_p5_sz16_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {1920; CHECK-LABEL: memcpy_p1_p5_sz16_align_2_2:1921; CHECK:       ; %bb.0: ; %entry1922; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1923; CHECK-NEXT:    s_clause 0x31924; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1925; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41926; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81927; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121928; CHECK-NEXT:    s_waitcnt vmcnt(0)1929; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1930; CHECK-NEXT:    s_setpc_b64 s[30:31]1931entry:1932  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)1933  ret void1934}1935 1936define void @memcpy_p1_p5_sz31_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {1937; CHECK-LABEL: memcpy_p1_p5_sz31_align_2_2:1938; CHECK:       ; %bb.0: ; %entry1939; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1940; CHECK-NEXT:    s_clause 0x71941; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen1942; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:41943; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen offset:81944; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:121945; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:161946; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:201947; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:231948; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:271949; CHECK-NEXT:    s_waitcnt vmcnt(6)1950; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[7:8], off1951; CHECK-NEXT:    s_waitcnt vmcnt(2)1952; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off offset:81953; CHECK-NEXT:    s_waitcnt vmcnt(0)1954; CHECK-NEXT:    global_store_dwordx2 v[0:1], v[9:10], off offset:231955; CHECK-NEXT:    s_setpc_b64 s[30:31]1956entry:1957  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)1958  ret void1959}1960 1961define void @memcpy_p1_p5_sz32_align_2_2(ptr addrspace(1) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {1962; CHECK-LABEL: memcpy_p1_p5_sz32_align_2_2:1963; CHECK:       ; %bb.0: ; %entry1964; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1965; CHECK-NEXT:    s_clause 0x71966; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1967; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41968; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81969; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121970; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:161971; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:201972; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:241973; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:281974; CHECK-NEXT:    s_waitcnt vmcnt(4)1975; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1976; CHECK-NEXT:    s_waitcnt vmcnt(0)1977; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:161978; CHECK-NEXT:    s_setpc_b64 s[30:31]1979entry:1980  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)1981  ret void1982}1983 1984define void @memcpy_p1_p5_sz16_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {1985; CHECK-LABEL: memcpy_p1_p5_sz16_align_8_8:1986; CHECK:       ; %bb.0: ; %entry1987; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1988; CHECK-NEXT:    s_clause 0x31989; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen1990; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:41991; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:81992; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:121993; CHECK-NEXT:    s_waitcnt vmcnt(0)1994; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off1995; CHECK-NEXT:    s_setpc_b64 s[30:31]1996entry:1997  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 16, i1 false)1998  ret void1999}2000 2001define void @memcpy_p1_p5_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {2002; CHECK-LABEL: memcpy_p1_p5_sz31_align_8_8:2003; CHECK:       ; %bb.0: ; %entry2004; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2005; CHECK-NEXT:    s_clause 0x72006; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen2007; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:42008; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:82009; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:122010; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:152011; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:192012; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:232013; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:272014; CHECK-NEXT:    s_waitcnt vmcnt(4)2015; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off2016; CHECK-NEXT:    s_waitcnt vmcnt(0)2017; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:152018; CHECK-NEXT:    s_setpc_b64 s[30:31]2019entry:2020  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 31, i1 false)2021  ret void2022}2023 2024define void @memcpy_p1_p5_sz32_align_8_8(ptr addrspace(1) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {2025; CHECK-LABEL: memcpy_p1_p5_sz32_align_8_8:2026; CHECK:       ; %bb.0: ; %entry2027; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2028; CHECK-NEXT:    s_clause 0x72029; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen2030; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:42031; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:82032; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:122033; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:162034; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:202035; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:242036; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:282037; CHECK-NEXT:    s_waitcnt vmcnt(4)2038; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off2039; CHECK-NEXT:    s_waitcnt vmcnt(0)2040; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:162041; CHECK-NEXT:    s_setpc_b64 s[30:31]2042entry:2043  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 32, i1 false)2044  ret void2045}2046 2047define void @memcpy_p1_p5_sz16_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {2048; CHECK-LABEL: memcpy_p1_p5_sz16_align_16_16:2049; CHECK:       ; %bb.0: ; %entry2050; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2051; CHECK-NEXT:    s_clause 0x32052; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen2053; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:42054; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:82055; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:122056; CHECK-NEXT:    s_waitcnt vmcnt(0)2057; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off2058; CHECK-NEXT:    s_setpc_b64 s[30:31]2059entry:2060  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 16, i1 false)2061  ret void2062}2063 2064define void @memcpy_p1_p5_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {2065; CHECK-LABEL: memcpy_p1_p5_sz31_align_16_16:2066; CHECK:       ; %bb.0: ; %entry2067; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2068; CHECK-NEXT:    s_clause 0x72069; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen2070; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:42071; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:82072; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:122073; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:152074; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:192075; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:232076; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:272077; CHECK-NEXT:    s_waitcnt vmcnt(4)2078; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off2079; CHECK-NEXT:    s_waitcnt vmcnt(0)2080; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:152081; CHECK-NEXT:    s_setpc_b64 s[30:31]2082entry:2083  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 31, i1 false)2084  ret void2085}2086 2087define void @memcpy_p1_p5_sz32_align_16_16(ptr addrspace(1) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {2088; CHECK-LABEL: memcpy_p1_p5_sz32_align_16_16:2089; CHECK:       ; %bb.0: ; %entry2090; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2091; CHECK-NEXT:    s_clause 0x72092; CHECK-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen2093; CHECK-NEXT:    buffer_load_dword v4, v2, s[0:3], 0 offen offset:42094; CHECK-NEXT:    buffer_load_dword v5, v2, s[0:3], 0 offen offset:82095; CHECK-NEXT:    buffer_load_dword v6, v2, s[0:3], 0 offen offset:122096; CHECK-NEXT:    buffer_load_dword v7, v2, s[0:3], 0 offen offset:162097; CHECK-NEXT:    buffer_load_dword v8, v2, s[0:3], 0 offen offset:202098; CHECK-NEXT:    buffer_load_dword v9, v2, s[0:3], 0 offen offset:242099; CHECK-NEXT:    buffer_load_dword v10, v2, s[0:3], 0 offen offset:282100; CHECK-NEXT:    s_waitcnt vmcnt(4)2101; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off2102; CHECK-NEXT:    s_waitcnt vmcnt(0)2103; CHECK-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off offset:162104; CHECK-NEXT:    s_setpc_b64 s[30:31]2105entry:2106  tail call void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 32, i1 false)2107  ret void2108}2109 2110define void @memcpy_p3_p0_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {2111; CHECK-LABEL: memcpy_p3_p0_sz16_align_1_1:2112; CHECK:       ; %bb.0: ; %entry2113; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2114; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]2115; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2116; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12117; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2118; CHECK-NEXT:    s_setpc_b64 s[30:31]2119entry:2120  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 16, i1 false)2121  ret void2122}2123 2124define void @memcpy_p3_p0_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {2125; CHECK-LABEL: memcpy_p3_p0_sz31_align_1_1:2126; CHECK:       ; %bb.0: ; %entry2127; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2128; CHECK-NEXT:    s_clause 0x22129; CHECK-NEXT:    flat_load_dwordx2 v[5:6], v[1:2] offset:232130; CHECK-NEXT:    flat_load_dwordx2 v[7:8], v[1:2] offset:162131; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]2132; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)2133; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:232134; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)2135; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162136; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)2137; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12138; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2139; CHECK-NEXT:    s_setpc_b64 s[30:31]2140entry:2141  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 31, i1 false)2142  ret void2143}2144 2145define void @memcpy_p3_p0_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {2146; CHECK-LABEL: memcpy_p3_p0_sz32_align_1_1:2147; CHECK:       ; %bb.0: ; %entry2148; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2149; CHECK-NEXT:    s_clause 0x12150; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:162151; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]2152; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)2153; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset0:2 offset1:32154; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)2155; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset1:12156; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2157; CHECK-NEXT:    s_setpc_b64 s[30:31]2158entry:2159  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)2160  ret void2161}2162 2163define void @memcpy_p3_p0_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {2164; CHECK-LABEL: memcpy_p3_p0_sz16_align_2_2:2165; CHECK:       ; %bb.0: ; %entry2166; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2167; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]2168; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2169; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12170; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2171; CHECK-NEXT:    s_setpc_b64 s[30:31]2172entry:2173  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 16, i1 false)2174  ret void2175}2176 2177define void @memcpy_p3_p0_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {2178; CHECK-LABEL: memcpy_p3_p0_sz31_align_2_2:2179; CHECK:       ; %bb.0: ; %entry2180; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2181; CHECK-NEXT:    s_clause 0x22182; CHECK-NEXT:    flat_load_dwordx2 v[5:6], v[1:2] offset:232183; CHECK-NEXT:    flat_load_dwordx2 v[7:8], v[1:2] offset:162184; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]2185; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)2186; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:232187; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(2)2188; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162189; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(2)2190; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12191; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2192; CHECK-NEXT:    s_setpc_b64 s[30:31]2193entry:2194  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 31, i1 false)2195  ret void2196}2197 2198define void @memcpy_p3_p0_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {2199; CHECK-LABEL: memcpy_p3_p0_sz32_align_2_2:2200; CHECK:       ; %bb.0: ; %entry2201; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2202; CHECK-NEXT:    s_clause 0x12203; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:162204; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]2205; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)2206; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset0:2 offset1:32207; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)2208; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset1:12209; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2210; CHECK-NEXT:    s_setpc_b64 s[30:31]2211entry:2212  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)2213  ret void2214}2215 2216define void @memcpy_p3_p0_sz16_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {2217; CHECK-LABEL: memcpy_p3_p0_sz16_align_8_8:2218; CHECK:       ; %bb.0: ; %entry2219; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2220; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]2221; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2222; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12223; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2224; CHECK-NEXT:    s_setpc_b64 s[30:31]2225entry:2226  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 16, i1 false)2227  ret void2228}2229 2230define void @memcpy_p3_p0_sz31_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {2231; CHECK-LABEL: memcpy_p3_p0_sz31_align_8_8:2232; CHECK:       ; %bb.0: ; %entry2233; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2234; CHECK-NEXT:    s_clause 0x12235; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2]2236; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2] offset:152237; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)2238; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12239; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)2240; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:152241; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2242; CHECK-NEXT:    s_setpc_b64 s[30:31]2243entry:2244  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 31, i1 false)2245  ret void2246}2247 2248define void @memcpy_p3_p0_sz32_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {2249; CHECK-LABEL: memcpy_p3_p0_sz32_align_8_8:2250; CHECK:       ; %bb.0: ; %entry2251; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2252; CHECK-NEXT:    s_clause 0x12253; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:162254; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]2255; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)2256; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset0:2 offset1:32257; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)2258; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset1:12259; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2260; CHECK-NEXT:    s_setpc_b64 s[30:31]2261entry:2262  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 32, i1 false)2263  ret void2264}2265 2266define void @memcpy_p3_p0_sz16_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {2267; CHECK-LABEL: memcpy_p3_p0_sz16_align_16_16:2268; CHECK:       ; %bb.0: ; %entry2269; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2270; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]2271; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2272; CHECK-NEXT:    ds_write_b128 v0, v[1:4]2273; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2274; CHECK-NEXT:    s_setpc_b64 s[30:31]2275entry:2276  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 16, i1 false)2277  ret void2278}2279 2280define void @memcpy_p3_p0_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {2281; CHECK-LABEL: memcpy_p3_p0_sz31_align_16_16:2282; CHECK:       ; %bb.0: ; %entry2283; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2284; CHECK-NEXT:    s_clause 0x12285; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:152286; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]2287; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)2288; CHECK-NEXT:    ds_write_b128 v0, v[3:6] offset:152289; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)2290; CHECK-NEXT:    ds_write_b128 v0, v[7:10]2291; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2292; CHECK-NEXT:    s_setpc_b64 s[30:31]2293entry:2294  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 31, i1 false)2295  ret void2296}2297 2298define void @memcpy_p3_p0_sz32_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {2299; CHECK-LABEL: memcpy_p3_p0_sz32_align_16_16:2300; CHECK:       ; %bb.0: ; %entry2301; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2302; CHECK-NEXT:    s_clause 0x12303; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:162304; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]2305; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)2306; CHECK-NEXT:    ds_write_b128 v0, v[3:6] offset:162307; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(1)2308; CHECK-NEXT:    ds_write_b128 v0, v[7:10]2309; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2310; CHECK-NEXT:    s_setpc_b64 s[30:31]2311entry:2312  tail call void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 32, i1 false)2313  ret void2314}2315 2316define void @memcpy_p3_p1_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {2317; CHECK-LABEL: memcpy_p3_p1_sz16_align_1_1:2318; CHECK:       ; %bb.0: ; %entry2319; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2320; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2321; CHECK-NEXT:    s_waitcnt vmcnt(0)2322; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12323; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2324; CHECK-NEXT:    s_setpc_b64 s[30:31]2325entry:2326  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 16, i1 false)2327  ret void2328}2329 2330define void @memcpy_p3_p1_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {2331; CHECK-LABEL: memcpy_p3_p1_sz31_align_1_1:2332; CHECK:       ; %bb.0: ; %entry2333; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2334; CHECK-NEXT:    s_clause 0x22335; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2336; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:162337; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:232338; CHECK-NEXT:    s_waitcnt vmcnt(2)2339; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12340; CHECK-NEXT:    s_waitcnt vmcnt(1)2341; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162342; CHECK-NEXT:    s_waitcnt vmcnt(0)2343; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:232344; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2345; CHECK-NEXT:    s_setpc_b64 s[30:31]2346entry:2347  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 31, i1 false)2348  ret void2349}2350 2351define void @memcpy_p3_p1_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {2352; CHECK-LABEL: memcpy_p3_p1_sz32_align_1_1:2353; CHECK:       ; %bb.0: ; %entry2354; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2355; CHECK-NEXT:    s_clause 0x12356; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2357; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162358; CHECK-NEXT:    s_waitcnt vmcnt(1)2359; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12360; CHECK-NEXT:    s_waitcnt vmcnt(0)2361; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset0:2 offset1:32362; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2363; CHECK-NEXT:    s_setpc_b64 s[30:31]2364entry:2365  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 32, i1 false)2366  ret void2367}2368 2369define void @memcpy_p3_p1_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {2370; CHECK-LABEL: memcpy_p3_p1_sz16_align_2_2:2371; CHECK:       ; %bb.0: ; %entry2372; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2373; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2374; CHECK-NEXT:    s_waitcnt vmcnt(0)2375; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12376; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2377; CHECK-NEXT:    s_setpc_b64 s[30:31]2378entry:2379  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 16, i1 false)2380  ret void2381}2382 2383define void @memcpy_p3_p1_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {2384; CHECK-LABEL: memcpy_p3_p1_sz31_align_2_2:2385; CHECK:       ; %bb.0: ; %entry2386; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2387; CHECK-NEXT:    s_clause 0x22388; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2389; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:162390; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:232391; CHECK-NEXT:    s_waitcnt vmcnt(2)2392; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12393; CHECK-NEXT:    s_waitcnt vmcnt(1)2394; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162395; CHECK-NEXT:    s_waitcnt vmcnt(0)2396; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:232397; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2398; CHECK-NEXT:    s_setpc_b64 s[30:31]2399entry:2400  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 31, i1 false)2401  ret void2402}2403 2404define void @memcpy_p3_p1_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {2405; CHECK-LABEL: memcpy_p3_p1_sz32_align_2_2:2406; CHECK:       ; %bb.0: ; %entry2407; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2408; CHECK-NEXT:    s_clause 0x12409; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2410; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162411; CHECK-NEXT:    s_waitcnt vmcnt(1)2412; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12413; CHECK-NEXT:    s_waitcnt vmcnt(0)2414; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset0:2 offset1:32415; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2416; CHECK-NEXT:    s_setpc_b64 s[30:31]2417entry:2418  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 32, i1 false)2419  ret void2420}2421 2422define void @memcpy_p3_p1_sz16_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {2423; CHECK-LABEL: memcpy_p3_p1_sz16_align_8_8:2424; CHECK:       ; %bb.0: ; %entry2425; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2426; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2427; CHECK-NEXT:    s_waitcnt vmcnt(0)2428; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12429; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2430; CHECK-NEXT:    s_setpc_b64 s[30:31]2431entry:2432  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 16, i1 false)2433  ret void2434}2435 2436define void @memcpy_p3_p1_sz31_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {2437; CHECK-LABEL: memcpy_p3_p1_sz31_align_8_8:2438; CHECK:       ; %bb.0: ; %entry2439; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2440; CHECK-NEXT:    s_clause 0x12441; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2442; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:152443; CHECK-NEXT:    s_waitcnt vmcnt(1)2444; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12445; CHECK-NEXT:    s_waitcnt vmcnt(0)2446; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:152447; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2448; CHECK-NEXT:    s_setpc_b64 s[30:31]2449entry:2450  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 31, i1 false)2451  ret void2452}2453 2454define void @memcpy_p3_p1_sz32_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {2455; CHECK-LABEL: memcpy_p3_p1_sz32_align_8_8:2456; CHECK:       ; %bb.0: ; %entry2457; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2458; CHECK-NEXT:    s_clause 0x12459; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2460; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162461; CHECK-NEXT:    s_waitcnt vmcnt(1)2462; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12463; CHECK-NEXT:    s_waitcnt vmcnt(0)2464; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset0:2 offset1:32465; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2466; CHECK-NEXT:    s_setpc_b64 s[30:31]2467entry:2468  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 32, i1 false)2469  ret void2470}2471 2472define void @memcpy_p3_p1_sz16_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {2473; CHECK-LABEL: memcpy_p3_p1_sz16_align_16_16:2474; CHECK:       ; %bb.0: ; %entry2475; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2476; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2477; CHECK-NEXT:    s_waitcnt vmcnt(0)2478; CHECK-NEXT:    ds_write_b128 v0, v[1:4]2479; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2480; CHECK-NEXT:    s_setpc_b64 s[30:31]2481entry:2482  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 16, i1 false)2483  ret void2484}2485 2486define void @memcpy_p3_p1_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {2487; CHECK-LABEL: memcpy_p3_p1_sz31_align_16_16:2488; CHECK:       ; %bb.0: ; %entry2489; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2490; CHECK-NEXT:    s_clause 0x12491; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2492; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:152493; CHECK-NEXT:    s_waitcnt vmcnt(1)2494; CHECK-NEXT:    ds_write_b128 v0, v[3:6]2495; CHECK-NEXT:    s_waitcnt vmcnt(0)2496; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:152497; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2498; CHECK-NEXT:    s_setpc_b64 s[30:31]2499entry:2500  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 31, i1 false)2501  ret void2502}2503 2504define void @memcpy_p3_p1_sz32_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {2505; CHECK-LABEL: memcpy_p3_p1_sz32_align_16_16:2506; CHECK:       ; %bb.0: ; %entry2507; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2508; CHECK-NEXT:    s_clause 0x12509; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2510; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162511; CHECK-NEXT:    s_waitcnt vmcnt(1)2512; CHECK-NEXT:    ds_write_b128 v0, v[3:6]2513; CHECK-NEXT:    s_waitcnt vmcnt(0)2514; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:162515; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2516; CHECK-NEXT:    s_setpc_b64 s[30:31]2517entry:2518  tail call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 32, i1 false)2519  ret void2520}2521 2522define void @memcpy_p3_p3_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {2523; CHECK-LABEL: memcpy_p3_p3_sz16_align_1_1:2524; CHECK:       ; %bb.0: ; %entry2525; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2526; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:12527; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2528; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12529; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2530; CHECK-NEXT:    s_setpc_b64 s[30:31]2531entry:2532  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 16, i1 false)2533  ret void2534}2535 2536define void @memcpy_p3_p3_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {2537; CHECK-LABEL: memcpy_p3_p3_sz31_align_1_1:2538; CHECK:       ; %bb.0: ; %entry2539; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2540; CHECK-NEXT:    ds_read_b64 v[5:6], v1 offset:232541; CHECK-NEXT:    ds_read_b64 v[7:8], v1 offset:162542; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:12543; CHECK-NEXT:    s_waitcnt lgkmcnt(2)2544; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:232545; CHECK-NEXT:    s_waitcnt lgkmcnt(2)2546; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162547; CHECK-NEXT:    s_waitcnt lgkmcnt(2)2548; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12549; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2550; CHECK-NEXT:    s_setpc_b64 s[30:31]2551entry:2552  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 31, i1 false)2553  ret void2554}2555 2556define void @memcpy_p3_p3_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {2557; CHECK-LABEL: memcpy_p3_p3_sz32_align_1_1:2558; CHECK:       ; %bb.0: ; %entry2559; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2560; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset0:2 offset1:32561; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset1:12562; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2563; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset0:2 offset1:32564; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2565; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset1:12566; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2567; CHECK-NEXT:    s_setpc_b64 s[30:31]2568entry:2569  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 32, i1 false)2570  ret void2571}2572 2573define void @memcpy_p3_p3_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {2574; CHECK-LABEL: memcpy_p3_p3_sz16_align_2_2:2575; CHECK:       ; %bb.0: ; %entry2576; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2577; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:12578; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2579; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12580; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2581; CHECK-NEXT:    s_setpc_b64 s[30:31]2582entry:2583  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 16, i1 false)2584  ret void2585}2586 2587define void @memcpy_p3_p3_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {2588; CHECK-LABEL: memcpy_p3_p3_sz31_align_2_2:2589; CHECK:       ; %bb.0: ; %entry2590; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2591; CHECK-NEXT:    ds_read_b64 v[5:6], v1 offset:232592; CHECK-NEXT:    ds_read_b64 v[7:8], v1 offset:162593; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:12594; CHECK-NEXT:    s_waitcnt lgkmcnt(2)2595; CHECK-NEXT:    ds_write_b64 v0, v[5:6] offset:232596; CHECK-NEXT:    s_waitcnt lgkmcnt(2)2597; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162598; CHECK-NEXT:    s_waitcnt lgkmcnt(2)2599; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12600; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2601; CHECK-NEXT:    s_setpc_b64 s[30:31]2602entry:2603  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 31, i1 false)2604  ret void2605}2606 2607define void @memcpy_p3_p3_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {2608; CHECK-LABEL: memcpy_p3_p3_sz32_align_2_2:2609; CHECK:       ; %bb.0: ; %entry2610; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2611; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset0:2 offset1:32612; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset1:12613; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2614; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset0:2 offset1:32615; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2616; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset1:12617; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2618; CHECK-NEXT:    s_setpc_b64 s[30:31]2619entry:2620  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 32, i1 false)2621  ret void2622}2623 2624define void @memcpy_p3_p3_sz16_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {2625; CHECK-LABEL: memcpy_p3_p3_sz16_align_8_8:2626; CHECK:       ; %bb.0: ; %entry2627; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2628; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:12629; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2630; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12631; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2632; CHECK-NEXT:    s_setpc_b64 s[30:31]2633entry:2634  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 16, i1 false)2635  ret void2636}2637 2638define void @memcpy_p3_p3_sz31_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {2639; CHECK-LABEL: memcpy_p3_p3_sz31_align_8_8:2640; CHECK:       ; %bb.0: ; %entry2641; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2642; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:12643; CHECK-NEXT:    ds_read_b128 v[6:9], v1 offset:152644; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2645; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:12646; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2647; CHECK-NEXT:    ds_write_b128 v0, v[6:9] offset:152648; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2649; CHECK-NEXT:    s_setpc_b64 s[30:31]2650entry:2651  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 31, i1 false)2652  ret void2653}2654 2655define void @memcpy_p3_p3_sz32_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {2656; CHECK-LABEL: memcpy_p3_p3_sz32_align_8_8:2657; CHECK:       ; %bb.0: ; %entry2658; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2659; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset0:2 offset1:32660; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset1:12661; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2662; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset0:2 offset1:32663; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2664; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset1:12665; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2666; CHECK-NEXT:    s_setpc_b64 s[30:31]2667entry:2668  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 32, i1 false)2669  ret void2670}2671 2672define void @memcpy_p3_p3_sz16_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {2673; CHECK-LABEL: memcpy_p3_p3_sz16_align_16_16:2674; CHECK:       ; %bb.0: ; %entry2675; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2676; CHECK-NEXT:    ds_read_b128 v[1:4], v12677; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2678; CHECK-NEXT:    ds_write_b128 v0, v[1:4]2679; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2680; CHECK-NEXT:    s_setpc_b64 s[30:31]2681entry:2682  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 16, i1 false)2683  ret void2684}2685 2686define void @memcpy_p3_p3_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {2687; CHECK-LABEL: memcpy_p3_p3_sz31_align_16_16:2688; CHECK:       ; %bb.0: ; %entry2689; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2690; CHECK-NEXT:    ds_read_b128 v[2:5], v1 offset:152691; CHECK-NEXT:    ds_read_b128 v[6:9], v12692; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2693; CHECK-NEXT:    ds_write_b128 v0, v[2:5] offset:152694; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2695; CHECK-NEXT:    ds_write_b128 v0, v[6:9]2696; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2697; CHECK-NEXT:    s_setpc_b64 s[30:31]2698entry:2699  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 31, i1 false)2700  ret void2701}2702 2703define void @memcpy_p3_p3_sz32_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {2704; CHECK-LABEL: memcpy_p3_p3_sz32_align_16_16:2705; CHECK:       ; %bb.0: ; %entry2706; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2707; CHECK-NEXT:    ds_read_b128 v[2:5], v1 offset:162708; CHECK-NEXT:    ds_read_b128 v[6:9], v12709; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2710; CHECK-NEXT:    ds_write_b128 v0, v[2:5] offset:162711; CHECK-NEXT:    s_waitcnt lgkmcnt(1)2712; CHECK-NEXT:    ds_write_b128 v0, v[6:9]2713; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2714; CHECK-NEXT:    s_setpc_b64 s[30:31]2715entry:2716  tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 32, i1 false)2717  ret void2718}2719 2720define void @memcpy_p3_p4_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {2721; CHECK-LABEL: memcpy_p3_p4_sz16_align_1_1:2722; CHECK:       ; %bb.0: ; %entry2723; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2724; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2725; CHECK-NEXT:    s_waitcnt vmcnt(0)2726; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12727; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2728; CHECK-NEXT:    s_setpc_b64 s[30:31]2729entry:2730  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 16, i1 false)2731  ret void2732}2733 2734define void @memcpy_p3_p4_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {2735; CHECK-LABEL: memcpy_p3_p4_sz31_align_1_1:2736; CHECK:       ; %bb.0: ; %entry2737; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2738; CHECK-NEXT:    s_clause 0x22739; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2740; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:162741; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:232742; CHECK-NEXT:    s_waitcnt vmcnt(2)2743; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12744; CHECK-NEXT:    s_waitcnt vmcnt(1)2745; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162746; CHECK-NEXT:    s_waitcnt vmcnt(0)2747; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:232748; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2749; CHECK-NEXT:    s_setpc_b64 s[30:31]2750entry:2751  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 31, i1 false)2752  ret void2753}2754 2755define void @memcpy_p3_p4_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {2756; CHECK-LABEL: memcpy_p3_p4_sz32_align_1_1:2757; CHECK:       ; %bb.0: ; %entry2758; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2759; CHECK-NEXT:    s_clause 0x12760; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2761; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162762; CHECK-NEXT:    s_waitcnt vmcnt(1)2763; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12764; CHECK-NEXT:    s_waitcnt vmcnt(0)2765; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset0:2 offset1:32766; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2767; CHECK-NEXT:    s_setpc_b64 s[30:31]2768entry:2769  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 32, i1 false)2770  ret void2771}2772 2773define void @memcpy_p3_p4_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {2774; CHECK-LABEL: memcpy_p3_p4_sz16_align_2_2:2775; CHECK:       ; %bb.0: ; %entry2776; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2777; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2778; CHECK-NEXT:    s_waitcnt vmcnt(0)2779; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12780; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2781; CHECK-NEXT:    s_setpc_b64 s[30:31]2782entry:2783  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 16, i1 false)2784  ret void2785}2786 2787define void @memcpy_p3_p4_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {2788; CHECK-LABEL: memcpy_p3_p4_sz31_align_2_2:2789; CHECK:       ; %bb.0: ; %entry2790; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2791; CHECK-NEXT:    s_clause 0x22792; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2793; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:162794; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:232795; CHECK-NEXT:    s_waitcnt vmcnt(2)2796; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12797; CHECK-NEXT:    s_waitcnt vmcnt(1)2798; CHECK-NEXT:    ds_write_b64 v0, v[7:8] offset:162799; CHECK-NEXT:    s_waitcnt vmcnt(0)2800; CHECK-NEXT:    ds_write_b64 v0, v[1:2] offset:232801; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2802; CHECK-NEXT:    s_setpc_b64 s[30:31]2803entry:2804  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 31, i1 false)2805  ret void2806}2807 2808define void @memcpy_p3_p4_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {2809; CHECK-LABEL: memcpy_p3_p4_sz32_align_2_2:2810; CHECK:       ; %bb.0: ; %entry2811; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2812; CHECK-NEXT:    s_clause 0x12813; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2814; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162815; CHECK-NEXT:    s_waitcnt vmcnt(1)2816; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12817; CHECK-NEXT:    s_waitcnt vmcnt(0)2818; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset0:2 offset1:32819; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2820; CHECK-NEXT:    s_setpc_b64 s[30:31]2821entry:2822  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 32, i1 false)2823  ret void2824}2825 2826define void @memcpy_p3_p4_sz16_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {2827; CHECK-LABEL: memcpy_p3_p4_sz16_align_8_8:2828; CHECK:       ; %bb.0: ; %entry2829; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2830; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2831; CHECK-NEXT:    s_waitcnt vmcnt(0)2832; CHECK-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:12833; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2834; CHECK-NEXT:    s_setpc_b64 s[30:31]2835entry:2836  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 16, i1 false)2837  ret void2838}2839 2840define void @memcpy_p3_p4_sz31_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {2841; CHECK-LABEL: memcpy_p3_p4_sz31_align_8_8:2842; CHECK:       ; %bb.0: ; %entry2843; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2844; CHECK-NEXT:    s_clause 0x12845; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2846; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:152847; CHECK-NEXT:    s_waitcnt vmcnt(1)2848; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12849; CHECK-NEXT:    s_waitcnt vmcnt(0)2850; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:152851; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2852; CHECK-NEXT:    s_setpc_b64 s[30:31]2853entry:2854  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 31, i1 false)2855  ret void2856}2857 2858define void @memcpy_p3_p4_sz32_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {2859; CHECK-LABEL: memcpy_p3_p4_sz32_align_8_8:2860; CHECK:       ; %bb.0: ; %entry2861; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2862; CHECK-NEXT:    s_clause 0x12863; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2864; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162865; CHECK-NEXT:    s_waitcnt vmcnt(1)2866; CHECK-NEXT:    ds_write2_b64 v0, v[3:4], v[5:6] offset1:12867; CHECK-NEXT:    s_waitcnt vmcnt(0)2868; CHECK-NEXT:    ds_write2_b64 v0, v[7:8], v[9:10] offset0:2 offset1:32869; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2870; CHECK-NEXT:    s_setpc_b64 s[30:31]2871entry:2872  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 32, i1 false)2873  ret void2874}2875 2876define void @memcpy_p3_p4_sz16_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {2877; CHECK-LABEL: memcpy_p3_p4_sz16_align_16_16:2878; CHECK:       ; %bb.0: ; %entry2879; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2880; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off2881; CHECK-NEXT:    s_waitcnt vmcnt(0)2882; CHECK-NEXT:    ds_write_b128 v0, v[1:4]2883; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2884; CHECK-NEXT:    s_setpc_b64 s[30:31]2885entry:2886  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 16, i1 false)2887  ret void2888}2889 2890define void @memcpy_p3_p4_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {2891; CHECK-LABEL: memcpy_p3_p4_sz31_align_16_16:2892; CHECK:       ; %bb.0: ; %entry2893; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2894; CHECK-NEXT:    s_clause 0x12895; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2896; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:152897; CHECK-NEXT:    s_waitcnt vmcnt(1)2898; CHECK-NEXT:    ds_write_b128 v0, v[3:6]2899; CHECK-NEXT:    s_waitcnt vmcnt(0)2900; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:152901; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2902; CHECK-NEXT:    s_setpc_b64 s[30:31]2903entry:2904  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 31, i1 false)2905  ret void2906}2907 2908define void @memcpy_p3_p4_sz32_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {2909; CHECK-LABEL: memcpy_p3_p4_sz32_align_16_16:2910; CHECK:       ; %bb.0: ; %entry2911; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2912; CHECK-NEXT:    s_clause 0x12913; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off2914; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:162915; CHECK-NEXT:    s_waitcnt vmcnt(1)2916; CHECK-NEXT:    ds_write_b128 v0, v[3:6]2917; CHECK-NEXT:    s_waitcnt vmcnt(0)2918; CHECK-NEXT:    ds_write_b128 v0, v[7:10] offset:162919; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2920; CHECK-NEXT:    s_setpc_b64 s[30:31]2921entry:2922  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 32, i1 false)2923  ret void2924}2925 2926define void @memcpy_p3_p5_sz16_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {2927; CHECK-LABEL: memcpy_p3_p5_sz16_align_1_1:2928; CHECK:       ; %bb.0: ; %entry2929; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2930; CHECK-NEXT:    s_clause 0x32931; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen2932; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:42933; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:82934; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:122935; CHECK-NEXT:    s_waitcnt vmcnt(0)2936; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:12937; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2938; CHECK-NEXT:    s_setpc_b64 s[30:31]2939entry:2940  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)2941  ret void2942}2943 2944define void @memcpy_p3_p5_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {2945; CHECK-LABEL: memcpy_p3_p5_sz31_align_1_1:2946; CHECK:       ; %bb.0: ; %entry2947; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2948; CHECK-NEXT:    s_clause 0x72949; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen2950; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:42951; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:82952; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:122953; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:162954; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:202955; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:232956; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:272957; CHECK-NEXT:    s_waitcnt vmcnt(4)2958; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:12959; CHECK-NEXT:    s_waitcnt vmcnt(2)2960; CHECK-NEXT:    ds_write_b64 v0, v[6:7] offset:162961; CHECK-NEXT:    s_waitcnt vmcnt(0)2962; CHECK-NEXT:    ds_write_b64 v0, v[8:9] offset:232963; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2964; CHECK-NEXT:    s_setpc_b64 s[30:31]2965entry:2966  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)2967  ret void2968}2969 2970define void @memcpy_p3_p5_sz32_align_1_1(ptr addrspace(3) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {2971; CHECK-LABEL: memcpy_p3_p5_sz32_align_1_1:2972; CHECK:       ; %bb.0: ; %entry2973; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2974; CHECK-NEXT:    s_clause 0x72975; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen2976; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:42977; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:82978; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:122979; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:162980; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:202981; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:242982; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:282983; CHECK-NEXT:    s_waitcnt vmcnt(4)2984; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:12985; CHECK-NEXT:    s_waitcnt vmcnt(0)2986; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset0:2 offset1:32987; CHECK-NEXT:    s_waitcnt lgkmcnt(0)2988; CHECK-NEXT:    s_setpc_b64 s[30:31]2989entry:2990  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)2991  ret void2992}2993 2994define void @memcpy_p3_p5_sz16_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {2995; CHECK-LABEL: memcpy_p3_p5_sz16_align_2_2:2996; CHECK:       ; %bb.0: ; %entry2997; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2998; CHECK-NEXT:    s_clause 0x32999; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3000; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43001; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83002; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123003; CHECK-NEXT:    s_waitcnt vmcnt(0)3004; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:13005; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3006; CHECK-NEXT:    s_setpc_b64 s[30:31]3007entry:3008  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)3009  ret void3010}3011 3012define void @memcpy_p3_p5_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {3013; CHECK-LABEL: memcpy_p3_p5_sz31_align_2_2:3014; CHECK:       ; %bb.0: ; %entry3015; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3016; CHECK-NEXT:    s_clause 0x73017; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3018; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43019; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83020; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123021; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:163022; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:203023; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:233024; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:273025; CHECK-NEXT:    s_waitcnt vmcnt(4)3026; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:13027; CHECK-NEXT:    s_waitcnt vmcnt(2)3028; CHECK-NEXT:    ds_write_b64 v0, v[6:7] offset:163029; CHECK-NEXT:    s_waitcnt vmcnt(0)3030; CHECK-NEXT:    ds_write_b64 v0, v[8:9] offset:233031; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3032; CHECK-NEXT:    s_setpc_b64 s[30:31]3033entry:3034  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)3035  ret void3036}3037 3038define void @memcpy_p3_p5_sz32_align_2_2(ptr addrspace(3) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {3039; CHECK-LABEL: memcpy_p3_p5_sz32_align_2_2:3040; CHECK:       ; %bb.0: ; %entry3041; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3042; CHECK-NEXT:    s_clause 0x73043; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3044; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43045; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83046; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123047; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:163048; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:203049; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:243050; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:283051; CHECK-NEXT:    s_waitcnt vmcnt(4)3052; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:13053; CHECK-NEXT:    s_waitcnt vmcnt(0)3054; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset0:2 offset1:33055; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3056; CHECK-NEXT:    s_setpc_b64 s[30:31]3057entry:3058  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)3059  ret void3060}3061 3062define void @memcpy_p3_p5_sz16_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {3063; CHECK-LABEL: memcpy_p3_p5_sz16_align_8_8:3064; CHECK:       ; %bb.0: ; %entry3065; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3066; CHECK-NEXT:    s_clause 0x33067; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3068; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43069; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83070; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123071; CHECK-NEXT:    s_waitcnt vmcnt(0)3072; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:13073; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3074; CHECK-NEXT:    s_setpc_b64 s[30:31]3075entry:3076  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 16, i1 false)3077  ret void3078}3079 3080define void @memcpy_p3_p5_sz31_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {3081; CHECK-LABEL: memcpy_p3_p5_sz31_align_8_8:3082; CHECK:       ; %bb.0: ; %entry3083; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3084; CHECK-NEXT:    s_clause 0x73085; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen3086; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:43087; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:83088; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:123089; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:153090; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:193091; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:233092; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:273093; CHECK-NEXT:    s_waitcnt vmcnt(4)3094; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset1:13095; CHECK-NEXT:    s_waitcnt vmcnt(0)3096; CHECK-NEXT:    ds_write_b128 v0, v[2:5] offset:153097; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3098; CHECK-NEXT:    s_setpc_b64 s[30:31]3099entry:3100  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 31, i1 false)3101  ret void3102}3103 3104define void @memcpy_p3_p5_sz32_align_8_8(ptr addrspace(3) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {3105; CHECK-LABEL: memcpy_p3_p5_sz32_align_8_8:3106; CHECK:       ; %bb.0: ; %entry3107; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3108; CHECK-NEXT:    s_clause 0x73109; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3110; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43111; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83112; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123113; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:163114; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:203115; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:243116; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:283117; CHECK-NEXT:    s_waitcnt vmcnt(4)3118; CHECK-NEXT:    ds_write2_b64 v0, v[2:3], v[4:5] offset1:13119; CHECK-NEXT:    s_waitcnt vmcnt(0)3120; CHECK-NEXT:    ds_write2_b64 v0, v[6:7], v[8:9] offset0:2 offset1:33121; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3122; CHECK-NEXT:    s_setpc_b64 s[30:31]3123entry:3124  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 32, i1 false)3125  ret void3126}3127 3128define void @memcpy_p3_p5_sz16_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {3129; CHECK-LABEL: memcpy_p3_p5_sz16_align_16_16:3130; CHECK:       ; %bb.0: ; %entry3131; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3132; CHECK-NEXT:    s_clause 0x33133; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3134; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43135; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83136; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123137; CHECK-NEXT:    s_waitcnt vmcnt(0)3138; CHECK-NEXT:    ds_write_b128 v0, v[2:5]3139; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3140; CHECK-NEXT:    s_setpc_b64 s[30:31]3141entry:3142  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 16, i1 false)3143  ret void3144}3145 3146define void @memcpy_p3_p5_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {3147; CHECK-LABEL: memcpy_p3_p5_sz31_align_16_16:3148; CHECK:       ; %bb.0: ; %entry3149; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3150; CHECK-NEXT:    s_clause 0x73151; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3152; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43153; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83154; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123155; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:153156; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:193157; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:233158; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:273159; CHECK-NEXT:    s_waitcnt vmcnt(4)3160; CHECK-NEXT:    ds_write_b128 v0, v[2:5]3161; CHECK-NEXT:    s_waitcnt vmcnt(0)3162; CHECK-NEXT:    ds_write_b128 v0, v[6:9] offset:153163; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3164; CHECK-NEXT:    s_setpc_b64 s[30:31]3165entry:3166  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 31, i1 false)3167  ret void3168}3169 3170define void @memcpy_p3_p5_sz32_align_16_16(ptr addrspace(3) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {3171; CHECK-LABEL: memcpy_p3_p5_sz32_align_16_16:3172; CHECK:       ; %bb.0: ; %entry3173; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3174; CHECK-NEXT:    s_clause 0x73175; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen3176; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:43177; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:83178; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:123179; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:163180; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:203181; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:243182; CHECK-NEXT:    buffer_load_dword v9, v1, s[0:3], 0 offen offset:283183; CHECK-NEXT:    s_waitcnt vmcnt(4)3184; CHECK-NEXT:    ds_write_b128 v0, v[2:5]3185; CHECK-NEXT:    s_waitcnt vmcnt(0)3186; CHECK-NEXT:    ds_write_b128 v0, v[6:9] offset:163187; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3188; CHECK-NEXT:    s_setpc_b64 s[30:31]3189entry:3190  tail call void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 32, i1 false)3191  ret void3192}3193 3194define void @memcpy_p5_p0_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {3195; CHECK-LABEL: memcpy_p5_p0_sz16_align_1_1:3196; CHECK:       ; %bb.0: ; %entry3197; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3198; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]3199; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3200; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123201; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83202; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43203; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3204; CHECK-NEXT:    s_setpc_b64 s[30:31]3205entry:3206  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 16, i1 false)3207  ret void3208}3209 3210define void @memcpy_p5_p0_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {3211; CHECK-LABEL: memcpy_p5_p0_sz31_align_1_1:3212; CHECK:       ; %bb.0: ; %entry3213; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3214; CHECK-NEXT:    s_clause 0x23215; CHECK-NEXT:    flat_load_dwordx2 v[5:6], v[1:2] offset:233216; CHECK-NEXT:    flat_load_dwordx2 v[7:8], v[1:2] offset:163217; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]3218; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)3219; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:273220; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:233221; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3222; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203223; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163224; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3225; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123226; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83227; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43228; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3229; CHECK-NEXT:    s_setpc_b64 s[30:31]3230entry:3231  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 31, i1 false)3232  ret void3233}3234 3235define void @memcpy_p5_p0_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align 1 readonly %src) {3236; CHECK-LABEL: memcpy_p5_p0_sz32_align_1_1:3237; CHECK:       ; %bb.0: ; %entry3238; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3239; CHECK-NEXT:    s_clause 0x13240; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:163241; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]3242; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3243; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:283244; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:243245; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:203246; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:163247; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3248; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:123249; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:83250; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:43251; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen3252; CHECK-NEXT:    s_setpc_b64 s[30:31]3253entry:3254  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 32, i1 false)3255  ret void3256}3257 3258define void @memcpy_p5_p0_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {3259; CHECK-LABEL: memcpy_p5_p0_sz16_align_2_2:3260; CHECK:       ; %bb.0: ; %entry3261; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3262; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]3263; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3264; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123265; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83266; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43267; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3268; CHECK-NEXT:    s_setpc_b64 s[30:31]3269entry:3270  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 16, i1 false)3271  ret void3272}3273 3274define void @memcpy_p5_p0_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {3275; CHECK-LABEL: memcpy_p5_p0_sz31_align_2_2:3276; CHECK:       ; %bb.0: ; %entry3277; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3278; CHECK-NEXT:    s_clause 0x23279; CHECK-NEXT:    flat_load_dwordx2 v[5:6], v[1:2] offset:233280; CHECK-NEXT:    flat_load_dwordx2 v[7:8], v[1:2] offset:163281; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]3282; CHECK-NEXT:    s_waitcnt vmcnt(2) lgkmcnt(2)3283; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:273284; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:233285; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3286; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203287; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163288; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3289; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123290; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83291; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43292; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3293; CHECK-NEXT:    s_setpc_b64 s[30:31]3294entry:3295  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 31, i1 false)3296  ret void3297}3298 3299define void @memcpy_p5_p0_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(0) align 2 readonly %src) {3300; CHECK-LABEL: memcpy_p5_p0_sz32_align_2_2:3301; CHECK:       ; %bb.0: ; %entry3302; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3303; CHECK-NEXT:    s_clause 0x13304; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:163305; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]3306; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3307; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:283308; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:243309; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:203310; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:163311; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3312; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:123313; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:83314; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:43315; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen3316; CHECK-NEXT:    s_setpc_b64 s[30:31]3317entry:3318  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(0) noundef nonnull align 2 %src, i64 32, i1 false)3319  ret void3320}3321 3322define void @memcpy_p5_p0_sz16_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {3323; CHECK-LABEL: memcpy_p5_p0_sz16_align_8_8:3324; CHECK:       ; %bb.0: ; %entry3325; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3326; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]3327; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3328; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123329; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83330; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43331; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3332; CHECK-NEXT:    s_setpc_b64 s[30:31]3333entry:3334  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 16, i1 false)3335  ret void3336}3337 3338define void @memcpy_p5_p0_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {3339; CHECK-LABEL: memcpy_p5_p0_sz31_align_8_8:3340; CHECK:       ; %bb.0: ; %entry3341; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3342; CHECK-NEXT:    s_clause 0x13343; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:153344; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]3345; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3346; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:273347; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:233348; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:193349; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:153350; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3351; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:123352; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:83353; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:43354; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen3355; CHECK-NEXT:    s_setpc_b64 s[30:31]3356entry:3357  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 31, i1 false)3358  ret void3359}3360 3361define void @memcpy_p5_p0_sz32_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(0) align 8 readonly %src) {3362; CHECK-LABEL: memcpy_p5_p0_sz32_align_8_8:3363; CHECK:       ; %bb.0: ; %entry3364; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3365; CHECK-NEXT:    s_clause 0x13366; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:163367; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]3368; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3369; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:283370; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:243371; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:203372; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:163373; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3374; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:123375; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:83376; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:43377; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen3378; CHECK-NEXT:    s_setpc_b64 s[30:31]3379entry:3380  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(0) noundef nonnull align 8 %src, i64 32, i1 false)3381  ret void3382}3383 3384define void @memcpy_p5_p0_sz16_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {3385; CHECK-LABEL: memcpy_p5_p0_sz16_align_16_16:3386; CHECK:       ; %bb.0: ; %entry3387; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3388; CHECK-NEXT:    flat_load_dwordx4 v[1:4], v[1:2]3389; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3390; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123391; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83392; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43393; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3394; CHECK-NEXT:    s_setpc_b64 s[30:31]3395entry:3396  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 16, i1 false)3397  ret void3398}3399 3400define void @memcpy_p5_p0_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {3401; CHECK-LABEL: memcpy_p5_p0_sz31_align_16_16:3402; CHECK:       ; %bb.0: ; %entry3403; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3404; CHECK-NEXT:    s_clause 0x13405; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:153406; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]3407; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3408; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:273409; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:233410; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:193411; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:153412; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3413; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:123414; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:83415; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:43416; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen3417; CHECK-NEXT:    s_setpc_b64 s[30:31]3418entry:3419  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 31, i1 false)3420  ret void3421}3422 3423define void @memcpy_p5_p0_sz32_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(0) align 16 readonly %src) {3424; CHECK-LABEL: memcpy_p5_p0_sz32_align_16_16:3425; CHECK:       ; %bb.0: ; %entry3426; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3427; CHECK-NEXT:    s_clause 0x13428; CHECK-NEXT:    flat_load_dwordx4 v[3:6], v[1:2] offset:163429; CHECK-NEXT:    flat_load_dwordx4 v[7:10], v[1:2]3430; CHECK-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)3431; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:283432; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:243433; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:203434; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:163435; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)3436; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:123437; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:83438; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:43439; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen3440; CHECK-NEXT:    s_setpc_b64 s[30:31]3441entry:3442  tail call void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(0) noundef nonnull align 16 %src, i64 32, i1 false)3443  ret void3444}3445 3446define void @memcpy_p5_p1_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {3447; CHECK-LABEL: memcpy_p5_p1_sz16_align_1_1:3448; CHECK:       ; %bb.0: ; %entry3449; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3450; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off3451; CHECK-NEXT:    s_waitcnt vmcnt(0)3452; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43453; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3454; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123455; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83456; CHECK-NEXT:    s_setpc_b64 s[30:31]3457entry:3458  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 16, i1 false)3459  ret void3460}3461 3462define void @memcpy_p5_p1_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {3463; CHECK-LABEL: memcpy_p5_p1_sz31_align_1_1:3464; CHECK:       ; %bb.0: ; %entry3465; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3466; CHECK-NEXT:    s_clause 0x23467; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3468; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:163469; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:233470; CHECK-NEXT:    s_waitcnt vmcnt(2)3471; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43472; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3473; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123474; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83475; CHECK-NEXT:    s_waitcnt vmcnt(1)3476; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203477; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163478; CHECK-NEXT:    s_waitcnt vmcnt(0)3479; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:273480; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:233481; CHECK-NEXT:    s_setpc_b64 s[30:31]3482entry:3483  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 31, i1 false)3484  ret void3485}3486 3487define void @memcpy_p5_p1_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align 1 readonly %src) {3488; CHECK-LABEL: memcpy_p5_p1_sz32_align_1_1:3489; CHECK:       ; %bb.0: ; %entry3490; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3491; CHECK-NEXT:    s_clause 0x13492; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3493; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:163494; CHECK-NEXT:    s_waitcnt vmcnt(1)3495; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43496; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3497; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123498; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83499; CHECK-NEXT:    s_waitcnt vmcnt(0)3500; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203501; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163502; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:283503; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:243504; CHECK-NEXT:    s_setpc_b64 s[30:31]3505entry:3506  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 32, i1 false)3507  ret void3508}3509 3510define void @memcpy_p5_p1_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {3511; CHECK-LABEL: memcpy_p5_p1_sz16_align_2_2:3512; CHECK:       ; %bb.0: ; %entry3513; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3514; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off3515; CHECK-NEXT:    s_waitcnt vmcnt(0)3516; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43517; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3518; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123519; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83520; CHECK-NEXT:    s_setpc_b64 s[30:31]3521entry:3522  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 16, i1 false)3523  ret void3524}3525 3526define void @memcpy_p5_p1_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {3527; CHECK-LABEL: memcpy_p5_p1_sz31_align_2_2:3528; CHECK:       ; %bb.0: ; %entry3529; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3530; CHECK-NEXT:    s_clause 0x23531; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3532; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:163533; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:233534; CHECK-NEXT:    s_waitcnt vmcnt(2)3535; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43536; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3537; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123538; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83539; CHECK-NEXT:    s_waitcnt vmcnt(1)3540; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203541; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163542; CHECK-NEXT:    s_waitcnt vmcnt(0)3543; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:273544; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:233545; CHECK-NEXT:    s_setpc_b64 s[30:31]3546entry:3547  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 31, i1 false)3548  ret void3549}3550 3551define void @memcpy_p5_p1_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(1) align 2 readonly %src) {3552; CHECK-LABEL: memcpy_p5_p1_sz32_align_2_2:3553; CHECK:       ; %bb.0: ; %entry3554; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3555; CHECK-NEXT:    s_clause 0x13556; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3557; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:163558; CHECK-NEXT:    s_waitcnt vmcnt(1)3559; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43560; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3561; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123562; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83563; CHECK-NEXT:    s_waitcnt vmcnt(0)3564; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203565; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163566; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:283567; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:243568; CHECK-NEXT:    s_setpc_b64 s[30:31]3569entry:3570  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(1) noundef nonnull align 2 %src, i64 32, i1 false)3571  ret void3572}3573 3574define void @memcpy_p5_p1_sz16_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {3575; CHECK-LABEL: memcpy_p5_p1_sz16_align_8_8:3576; CHECK:       ; %bb.0: ; %entry3577; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3578; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off3579; CHECK-NEXT:    s_waitcnt vmcnt(0)3580; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123581; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83582; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43583; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3584; CHECK-NEXT:    s_setpc_b64 s[30:31]3585entry:3586  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 16, i1 false)3587  ret void3588}3589 3590define void @memcpy_p5_p1_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {3591; CHECK-LABEL: memcpy_p5_p1_sz31_align_8_8:3592; CHECK:       ; %bb.0: ; %entry3593; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3594; CHECK-NEXT:    s_clause 0x13595; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3596; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:153597; CHECK-NEXT:    s_waitcnt vmcnt(1)3598; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123599; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83600; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43601; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3602; CHECK-NEXT:    s_waitcnt vmcnt(0)3603; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:273604; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:233605; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:193606; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:153607; CHECK-NEXT:    s_setpc_b64 s[30:31]3608entry:3609  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 31, i1 false)3610  ret void3611}3612 3613define void @memcpy_p5_p1_sz32_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(1) align 8 readonly %src) {3614; CHECK-LABEL: memcpy_p5_p1_sz32_align_8_8:3615; CHECK:       ; %bb.0: ; %entry3616; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3617; CHECK-NEXT:    s_clause 0x13618; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3619; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:163620; CHECK-NEXT:    s_waitcnt vmcnt(1)3621; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123622; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83623; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43624; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3625; CHECK-NEXT:    s_waitcnt vmcnt(0)3626; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:283627; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:243628; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203629; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163630; CHECK-NEXT:    s_setpc_b64 s[30:31]3631entry:3632  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(1) noundef nonnull align 8 %src, i64 32, i1 false)3633  ret void3634}3635 3636define void @memcpy_p5_p1_sz16_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {3637; CHECK-LABEL: memcpy_p5_p1_sz16_align_16_16:3638; CHECK:       ; %bb.0: ; %entry3639; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3640; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off3641; CHECK-NEXT:    s_waitcnt vmcnt(0)3642; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123643; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83644; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43645; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3646; CHECK-NEXT:    s_setpc_b64 s[30:31]3647entry:3648  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 16, i1 false)3649  ret void3650}3651 3652define void @memcpy_p5_p1_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {3653; CHECK-LABEL: memcpy_p5_p1_sz31_align_16_16:3654; CHECK:       ; %bb.0: ; %entry3655; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3656; CHECK-NEXT:    s_clause 0x13657; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3658; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:153659; CHECK-NEXT:    s_waitcnt vmcnt(1)3660; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123661; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83662; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43663; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3664; CHECK-NEXT:    s_waitcnt vmcnt(0)3665; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:273666; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:233667; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:193668; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:153669; CHECK-NEXT:    s_setpc_b64 s[30:31]3670entry:3671  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 31, i1 false)3672  ret void3673}3674 3675define void @memcpy_p5_p1_sz32_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(1) align 16 readonly %src) {3676; CHECK-LABEL: memcpy_p5_p1_sz32_align_16_16:3677; CHECK:       ; %bb.0: ; %entry3678; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3679; CHECK-NEXT:    s_clause 0x13680; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3681; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:163682; CHECK-NEXT:    s_waitcnt vmcnt(1)3683; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123684; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83685; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43686; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3687; CHECK-NEXT:    s_waitcnt vmcnt(0)3688; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:283689; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:243690; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203691; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163692; CHECK-NEXT:    s_setpc_b64 s[30:31]3693entry:3694  tail call void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(1) noundef nonnull align 16 %src, i64 32, i1 false)3695  ret void3696}3697 3698define void @memcpy_p5_p3_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {3699; CHECK-LABEL: memcpy_p5_p3_sz16_align_1_1:3700; CHECK:       ; %bb.0: ; %entry3701; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3702; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:13703; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3704; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43705; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3706; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123707; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83708; CHECK-NEXT:    s_setpc_b64 s[30:31]3709entry:3710  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 16, i1 false)3711  ret void3712}3713 3714define void @memcpy_p5_p3_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {3715; CHECK-LABEL: memcpy_p5_p3_sz31_align_1_1:3716; CHECK:       ; %bb.0: ; %entry3717; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3718; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:13719; CHECK-NEXT:    ds_read_b64 v[6:7], v1 offset:163720; CHECK-NEXT:    ds_read_b64 v[8:9], v1 offset:233721; CHECK-NEXT:    s_waitcnt lgkmcnt(2)3722; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43723; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3724; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123725; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83726; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3727; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:203728; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:163729; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3730; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:273731; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:233732; CHECK-NEXT:    s_setpc_b64 s[30:31]3733entry:3734  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 31, i1 false)3735  ret void3736}3737 3738define void @memcpy_p5_p3_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align 1 readonly %src) {3739; CHECK-LABEL: memcpy_p5_p3_sz32_align_1_1:3740; CHECK:       ; %bb.0: ; %entry3741; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3742; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:13743; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset0:2 offset1:33744; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3745; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43746; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3747; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123748; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83749; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3750; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:203751; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:163752; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:283753; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:243754; CHECK-NEXT:    s_setpc_b64 s[30:31]3755entry:3756  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 32, i1 false)3757  ret void3758}3759 3760define void @memcpy_p5_p3_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {3761; CHECK-LABEL: memcpy_p5_p3_sz16_align_2_2:3762; CHECK:       ; %bb.0: ; %entry3763; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3764; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:13765; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3766; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43767; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3768; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123769; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83770; CHECK-NEXT:    s_setpc_b64 s[30:31]3771entry:3772  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 16, i1 false)3773  ret void3774}3775 3776define void @memcpy_p5_p3_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {3777; CHECK-LABEL: memcpy_p5_p3_sz31_align_2_2:3778; CHECK:       ; %bb.0: ; %entry3779; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3780; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:13781; CHECK-NEXT:    ds_read_b64 v[6:7], v1 offset:163782; CHECK-NEXT:    ds_read_b64 v[8:9], v1 offset:233783; CHECK-NEXT:    s_waitcnt lgkmcnt(2)3784; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43785; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3786; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123787; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83788; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3789; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:203790; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:163791; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3792; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:273793; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:233794; CHECK-NEXT:    s_setpc_b64 s[30:31]3795entry:3796  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 31, i1 false)3797  ret void3798}3799 3800define void @memcpy_p5_p3_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(3) align 2 readonly %src) {3801; CHECK-LABEL: memcpy_p5_p3_sz32_align_2_2:3802; CHECK:       ; %bb.0: ; %entry3803; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3804; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:13805; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset0:2 offset1:33806; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3807; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43808; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3809; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123810; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83811; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3812; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:203813; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:163814; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:283815; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:243816; CHECK-NEXT:    s_setpc_b64 s[30:31]3817entry:3818  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(3) noundef nonnull align 2 %src, i64 32, i1 false)3819  ret void3820}3821 3822define void @memcpy_p5_p3_sz16_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {3823; CHECK-LABEL: memcpy_p5_p3_sz16_align_8_8:3824; CHECK:       ; %bb.0: ; %entry3825; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3826; CHECK-NEXT:    ds_read2_b64 v[1:4], v1 offset1:13827; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3828; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123829; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83830; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43831; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3832; CHECK-NEXT:    s_setpc_b64 s[30:31]3833entry:3834  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 16, i1 false)3835  ret void3836}3837 3838define void @memcpy_p5_p3_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {3839; CHECK-LABEL: memcpy_p5_p3_sz31_align_8_8:3840; CHECK:       ; %bb.0: ; %entry3841; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3842; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:13843; CHECK-NEXT:    ds_read_b128 v[6:9], v1 offset:153844; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3845; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123846; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83847; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43848; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3849; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3850; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:273851; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:233852; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:193853; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:153854; CHECK-NEXT:    s_setpc_b64 s[30:31]3855entry:3856  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 31, i1 false)3857  ret void3858}3859 3860define void @memcpy_p5_p3_sz32_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(3) align 8 readonly %src) {3861; CHECK-LABEL: memcpy_p5_p3_sz32_align_8_8:3862; CHECK:       ; %bb.0: ; %entry3863; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3864; CHECK-NEXT:    ds_read2_b64 v[2:5], v1 offset1:13865; CHECK-NEXT:    ds_read2_b64 v[6:9], v1 offset0:2 offset1:33866; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3867; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123868; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83869; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43870; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3871; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3872; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:283873; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:243874; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:203875; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:163876; CHECK-NEXT:    s_setpc_b64 s[30:31]3877entry:3878  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 32, i1 false)3879  ret void3880}3881 3882define void @memcpy_p5_p3_sz16_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {3883; CHECK-LABEL: memcpy_p5_p3_sz16_align_16_16:3884; CHECK:       ; %bb.0: ; %entry3885; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3886; CHECK-NEXT:    ds_read_b128 v[1:4], v13887; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3888; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123889; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83890; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43891; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3892; CHECK-NEXT:    s_setpc_b64 s[30:31]3893entry:3894  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 16, i1 false)3895  ret void3896}3897 3898define void @memcpy_p5_p3_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {3899; CHECK-LABEL: memcpy_p5_p3_sz31_align_16_16:3900; CHECK:       ; %bb.0: ; %entry3901; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3902; CHECK-NEXT:    ds_read_b128 v[2:5], v13903; CHECK-NEXT:    ds_read_b128 v[6:9], v1 offset:153904; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3905; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123906; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83907; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43908; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3909; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3910; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:273911; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:233912; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:193913; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:153914; CHECK-NEXT:    s_setpc_b64 s[30:31]3915entry:3916  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 31, i1 false)3917  ret void3918}3919 3920define void @memcpy_p5_p3_sz32_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(3) align 16 readonly %src) {3921; CHECK-LABEL: memcpy_p5_p3_sz32_align_16_16:3922; CHECK:       ; %bb.0: ; %entry3923; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3924; CHECK-NEXT:    ds_read_b128 v[2:5], v13925; CHECK-NEXT:    ds_read_b128 v[6:9], v1 offset:163926; CHECK-NEXT:    s_waitcnt lgkmcnt(1)3927; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:123928; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:83929; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:43930; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen3931; CHECK-NEXT:    s_waitcnt lgkmcnt(0)3932; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:283933; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:243934; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:203935; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:163936; CHECK-NEXT:    s_setpc_b64 s[30:31]3937entry:3938  tail call void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 32, i1 false)3939  ret void3940}3941 3942define void @memcpy_p5_p4_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {3943; CHECK-LABEL: memcpy_p5_p4_sz16_align_1_1:3944; CHECK:       ; %bb.0: ; %entry3945; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3946; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off3947; CHECK-NEXT:    s_waitcnt vmcnt(0)3948; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:43949; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen3950; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:123951; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:83952; CHECK-NEXT:    s_setpc_b64 s[30:31]3953entry:3954  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 16, i1 false)3955  ret void3956}3957 3958define void @memcpy_p5_p4_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {3959; CHECK-LABEL: memcpy_p5_p4_sz31_align_1_1:3960; CHECK:       ; %bb.0: ; %entry3961; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3962; CHECK-NEXT:    s_clause 0x23963; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3964; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:163965; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:233966; CHECK-NEXT:    s_waitcnt vmcnt(2)3967; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43968; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3969; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123970; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83971; CHECK-NEXT:    s_waitcnt vmcnt(1)3972; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203973; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163974; CHECK-NEXT:    s_waitcnt vmcnt(0)3975; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:273976; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:233977; CHECK-NEXT:    s_setpc_b64 s[30:31]3978entry:3979  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 31, i1 false)3980  ret void3981}3982 3983define void @memcpy_p5_p4_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {3984; CHECK-LABEL: memcpy_p5_p4_sz32_align_1_1:3985; CHECK:       ; %bb.0: ; %entry3986; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3987; CHECK-NEXT:    s_clause 0x13988; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off3989; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:163990; CHECK-NEXT:    s_waitcnt vmcnt(1)3991; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:43992; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen3993; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:123994; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:83995; CHECK-NEXT:    s_waitcnt vmcnt(0)3996; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:203997; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:163998; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:283999; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:244000; CHECK-NEXT:    s_setpc_b64 s[30:31]4001entry:4002  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 32, i1 false)4003  ret void4004}4005 4006define void @memcpy_p5_p4_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {4007; CHECK-LABEL: memcpy_p5_p4_sz16_align_2_2:4008; CHECK:       ; %bb.0: ; %entry4009; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4010; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off4011; CHECK-NEXT:    s_waitcnt vmcnt(0)4012; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:44013; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen4014; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:124015; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:84016; CHECK-NEXT:    s_setpc_b64 s[30:31]4017entry:4018  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 16, i1 false)4019  ret void4020}4021 4022define void @memcpy_p5_p4_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {4023; CHECK-LABEL: memcpy_p5_p4_sz31_align_2_2:4024; CHECK:       ; %bb.0: ; %entry4025; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4026; CHECK-NEXT:    s_clause 0x24027; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off4028; CHECK-NEXT:    global_load_dwordx2 v[7:8], v[1:2], off offset:164029; CHECK-NEXT:    global_load_dwordx2 v[1:2], v[1:2], off offset:234030; CHECK-NEXT:    s_waitcnt vmcnt(2)4031; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:44032; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen4033; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:124034; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:84035; CHECK-NEXT:    s_waitcnt vmcnt(1)4036; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:204037; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:164038; CHECK-NEXT:    s_waitcnt vmcnt(0)4039; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:274040; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:234041; CHECK-NEXT:    s_setpc_b64 s[30:31]4042entry:4043  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 31, i1 false)4044  ret void4045}4046 4047define void @memcpy_p5_p4_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(4) align 2 readonly %src) {4048; CHECK-LABEL: memcpy_p5_p4_sz32_align_2_2:4049; CHECK:       ; %bb.0: ; %entry4050; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4051; CHECK-NEXT:    s_clause 0x14052; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off4053; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:164054; CHECK-NEXT:    s_waitcnt vmcnt(1)4055; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:44056; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen4057; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:124058; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:84059; CHECK-NEXT:    s_waitcnt vmcnt(0)4060; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:204061; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:164062; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:284063; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:244064; CHECK-NEXT:    s_setpc_b64 s[30:31]4065entry:4066  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(4) noundef nonnull align 2 %src, i64 32, i1 false)4067  ret void4068}4069 4070define void @memcpy_p5_p4_sz16_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {4071; CHECK-LABEL: memcpy_p5_p4_sz16_align_8_8:4072; CHECK:       ; %bb.0: ; %entry4073; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4074; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off4075; CHECK-NEXT:    s_waitcnt vmcnt(0)4076; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:124077; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:84078; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:44079; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen4080; CHECK-NEXT:    s_setpc_b64 s[30:31]4081entry:4082  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 16, i1 false)4083  ret void4084}4085 4086define void @memcpy_p5_p4_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {4087; CHECK-LABEL: memcpy_p5_p4_sz31_align_8_8:4088; CHECK:       ; %bb.0: ; %entry4089; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4090; CHECK-NEXT:    s_clause 0x14091; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off4092; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:154093; CHECK-NEXT:    s_waitcnt vmcnt(1)4094; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:124095; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:84096; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:44097; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen4098; CHECK-NEXT:    s_waitcnt vmcnt(0)4099; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:274100; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:234101; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:194102; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:154103; CHECK-NEXT:    s_setpc_b64 s[30:31]4104entry:4105  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 31, i1 false)4106  ret void4107}4108 4109define void @memcpy_p5_p4_sz32_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(4) align 8 readonly %src) {4110; CHECK-LABEL: memcpy_p5_p4_sz32_align_8_8:4111; CHECK:       ; %bb.0: ; %entry4112; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4113; CHECK-NEXT:    s_clause 0x14114; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off4115; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:164116; CHECK-NEXT:    s_waitcnt vmcnt(1)4117; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:124118; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:84119; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:44120; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen4121; CHECK-NEXT:    s_waitcnt vmcnt(0)4122; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:284123; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:244124; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:204125; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:164126; CHECK-NEXT:    s_setpc_b64 s[30:31]4127entry:4128  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(4) noundef nonnull align 8 %src, i64 32, i1 false)4129  ret void4130}4131 4132define void @memcpy_p5_p4_sz16_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {4133; CHECK-LABEL: memcpy_p5_p4_sz16_align_16_16:4134; CHECK:       ; %bb.0: ; %entry4135; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4136; CHECK-NEXT:    global_load_dwordx4 v[1:4], v[1:2], off4137; CHECK-NEXT:    s_waitcnt vmcnt(0)4138; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:124139; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:84140; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:44141; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen4142; CHECK-NEXT:    s_setpc_b64 s[30:31]4143entry:4144  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 16, i1 false)4145  ret void4146}4147 4148define void @memcpy_p5_p4_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {4149; CHECK-LABEL: memcpy_p5_p4_sz31_align_16_16:4150; CHECK:       ; %bb.0: ; %entry4151; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4152; CHECK-NEXT:    s_clause 0x14153; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off4154; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:154155; CHECK-NEXT:    s_waitcnt vmcnt(1)4156; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:124157; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:84158; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:44159; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen4160; CHECK-NEXT:    s_waitcnt vmcnt(0)4161; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:274162; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:234163; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:194164; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:154165; CHECK-NEXT:    s_setpc_b64 s[30:31]4166entry:4167  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 31, i1 false)4168  ret void4169}4170 4171define void @memcpy_p5_p4_sz32_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(4) align 16 readonly %src) {4172; CHECK-LABEL: memcpy_p5_p4_sz32_align_16_16:4173; CHECK:       ; %bb.0: ; %entry4174; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4175; CHECK-NEXT:    s_clause 0x14176; CHECK-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off4177; CHECK-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off offset:164178; CHECK-NEXT:    s_waitcnt vmcnt(1)4179; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:124180; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:84181; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:44182; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen4183; CHECK-NEXT:    s_waitcnt vmcnt(0)4184; CHECK-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:284185; CHECK-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:244186; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:204187; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:164188; CHECK-NEXT:    s_setpc_b64 s[30:31]4189entry:4190  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(4) noundef nonnull align 16 %src, i64 32, i1 false)4191  ret void4192}4193 4194define void @memcpy_p5_p5_sz16_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {4195; CHECK-LABEL: memcpy_p5_p5_sz16_align_1_1:4196; CHECK:       ; %bb.0: ; %entry4197; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4198; CHECK-NEXT:    s_clause 0x34199; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:84200; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:124201; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen4202; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:44203; CHECK-NEXT:    s_waitcnt vmcnt(3)4204; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:84205; CHECK-NEXT:    s_waitcnt vmcnt(2)4206; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:124207; CHECK-NEXT:    s_waitcnt vmcnt(1)4208; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen4209; CHECK-NEXT:    s_waitcnt vmcnt(0)4210; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:44211; CHECK-NEXT:    s_setpc_b64 s[30:31]4212entry:4213  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 16, i1 false)4214  ret void4215}4216 4217define void @memcpy_p5_p5_sz31_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {4218; CHECK-LABEL: memcpy_p5_p5_sz31_align_1_1:4219; CHECK:       ; %bb.0: ; %entry4220; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4221; CHECK-NEXT:    s_clause 0x74222; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:234223; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:274224; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:164225; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:204226; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:84227; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:124228; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen4229; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:44230; CHECK-NEXT:    s_waitcnt vmcnt(7)4231; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:234232; CHECK-NEXT:    s_waitcnt vmcnt(6)4233; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:274234; CHECK-NEXT:    s_waitcnt vmcnt(5)4235; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:164236; CHECK-NEXT:    s_waitcnt vmcnt(4)4237; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:204238; CHECK-NEXT:    s_waitcnt vmcnt(3)4239; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:84240; CHECK-NEXT:    s_waitcnt vmcnt(2)4241; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:124242; CHECK-NEXT:    s_waitcnt vmcnt(1)4243; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen4244; CHECK-NEXT:    s_waitcnt vmcnt(0)4245; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:44246; CHECK-NEXT:    s_setpc_b64 s[30:31]4247entry:4248  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 31, i1 false)4249  ret void4250}4251 4252define void @memcpy_p5_p5_sz32_align_1_1(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align 1 readonly %src) {4253; CHECK-LABEL: memcpy_p5_p5_sz32_align_1_1:4254; CHECK:       ; %bb.0: ; %entry4255; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4256; CHECK-NEXT:    s_clause 0x74257; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:244258; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:284259; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:164260; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:204261; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:84262; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:124263; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen4264; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:44265; CHECK-NEXT:    s_waitcnt vmcnt(7)4266; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:244267; CHECK-NEXT:    s_waitcnt vmcnt(6)4268; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:284269; CHECK-NEXT:    s_waitcnt vmcnt(5)4270; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:164271; CHECK-NEXT:    s_waitcnt vmcnt(4)4272; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:204273; CHECK-NEXT:    s_waitcnt vmcnt(3)4274; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:84275; CHECK-NEXT:    s_waitcnt vmcnt(2)4276; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:124277; CHECK-NEXT:    s_waitcnt vmcnt(1)4278; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen4279; CHECK-NEXT:    s_waitcnt vmcnt(0)4280; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:44281; CHECK-NEXT:    s_setpc_b64 s[30:31]4282entry:4283  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 32, i1 false)4284  ret void4285}4286 4287define void @memcpy_p5_p5_sz16_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {4288; CHECK-LABEL: memcpy_p5_p5_sz16_align_2_2:4289; CHECK:       ; %bb.0: ; %entry4290; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4291; CHECK-NEXT:    s_clause 0x34292; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:84293; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:124294; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen4295; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:44296; CHECK-NEXT:    s_waitcnt vmcnt(3)4297; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:84298; CHECK-NEXT:    s_waitcnt vmcnt(2)4299; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:124300; CHECK-NEXT:    s_waitcnt vmcnt(1)4301; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen4302; CHECK-NEXT:    s_waitcnt vmcnt(0)4303; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:44304; CHECK-NEXT:    s_setpc_b64 s[30:31]4305entry:4306  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 16, i1 false)4307  ret void4308}4309 4310define void @memcpy_p5_p5_sz31_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {4311; CHECK-LABEL: memcpy_p5_p5_sz31_align_2_2:4312; CHECK:       ; %bb.0: ; %entry4313; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4314; CHECK-NEXT:    s_clause 0x74315; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:234316; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:274317; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:164318; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:204319; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:84320; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:124321; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen4322; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:44323; CHECK-NEXT:    s_waitcnt vmcnt(7)4324; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:234325; CHECK-NEXT:    s_waitcnt vmcnt(6)4326; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:274327; CHECK-NEXT:    s_waitcnt vmcnt(5)4328; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:164329; CHECK-NEXT:    s_waitcnt vmcnt(4)4330; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:204331; CHECK-NEXT:    s_waitcnt vmcnt(3)4332; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:84333; CHECK-NEXT:    s_waitcnt vmcnt(2)4334; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:124335; CHECK-NEXT:    s_waitcnt vmcnt(1)4336; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen4337; CHECK-NEXT:    s_waitcnt vmcnt(0)4338; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:44339; CHECK-NEXT:    s_setpc_b64 s[30:31]4340entry:4341  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 31, i1 false)4342  ret void4343}4344 4345define void @memcpy_p5_p5_sz32_align_2_2(ptr addrspace(5) align 2 %dst, ptr addrspace(5) align 2 readonly %src) {4346; CHECK-LABEL: memcpy_p5_p5_sz32_align_2_2:4347; CHECK:       ; %bb.0: ; %entry4348; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4349; CHECK-NEXT:    s_clause 0x74350; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:244351; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:284352; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:164353; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:204354; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen offset:84355; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:124356; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen4357; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:44358; CHECK-NEXT:    s_waitcnt vmcnt(7)4359; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:244360; CHECK-NEXT:    s_waitcnt vmcnt(6)4361; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:284362; CHECK-NEXT:    s_waitcnt vmcnt(5)4363; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:164364; CHECK-NEXT:    s_waitcnt vmcnt(4)4365; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:204366; CHECK-NEXT:    s_waitcnt vmcnt(3)4367; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:84368; CHECK-NEXT:    s_waitcnt vmcnt(2)4369; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:124370; CHECK-NEXT:    s_waitcnt vmcnt(1)4371; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen4372; CHECK-NEXT:    s_waitcnt vmcnt(0)4373; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:44374; CHECK-NEXT:    s_setpc_b64 s[30:31]4375entry:4376  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 2 %dst, ptr addrspace(5) noundef nonnull align 2 %src, i64 32, i1 false)4377  ret void4378}4379 4380define void @memcpy_p5_p5_sz16_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {4381; CHECK-LABEL: memcpy_p5_p5_sz16_align_8_8:4382; CHECK:       ; %bb.0: ; %entry4383; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4384; CHECK-NEXT:    s_clause 0x34385; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen4386; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:44387; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:84388; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:124389; CHECK-NEXT:    s_waitcnt vmcnt(3)4390; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen4391; CHECK-NEXT:    s_waitcnt vmcnt(2)4392; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:44393; CHECK-NEXT:    s_waitcnt vmcnt(1)4394; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:84395; CHECK-NEXT:    s_waitcnt vmcnt(0)4396; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124397; CHECK-NEXT:    s_setpc_b64 s[30:31]4398entry:4399  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 16, i1 false)4400  ret void4401}4402 4403define void @memcpy_p5_p5_sz31_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {4404; CHECK-LABEL: memcpy_p5_p5_sz31_align_8_8:4405; CHECK:       ; %bb.0: ; %entry4406; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4407; CHECK-NEXT:    s_clause 0x74408; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:154409; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:194410; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:234411; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:274412; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen4413; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:44414; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:84415; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:124416; CHECK-NEXT:    s_waitcnt vmcnt(7)4417; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:154418; CHECK-NEXT:    s_waitcnt vmcnt(6)4419; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:194420; CHECK-NEXT:    s_waitcnt vmcnt(5)4421; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:234422; CHECK-NEXT:    s_waitcnt vmcnt(4)4423; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:274424; CHECK-NEXT:    s_waitcnt vmcnt(3)4425; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen4426; CHECK-NEXT:    s_waitcnt vmcnt(2)4427; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:44428; CHECK-NEXT:    s_waitcnt vmcnt(1)4429; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:84430; CHECK-NEXT:    s_waitcnt vmcnt(0)4431; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124432; CHECK-NEXT:    s_setpc_b64 s[30:31]4433entry:4434  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 31, i1 false)4435  ret void4436}4437 4438define void @memcpy_p5_p5_sz32_align_8_8(ptr addrspace(5) align 8 %dst, ptr addrspace(5) align 8 readonly %src) {4439; CHECK-LABEL: memcpy_p5_p5_sz32_align_8_8:4440; CHECK:       ; %bb.0: ; %entry4441; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4442; CHECK-NEXT:    s_clause 0x74443; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:164444; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:204445; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:244446; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:284447; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen4448; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:44449; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:84450; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:124451; CHECK-NEXT:    s_waitcnt vmcnt(7)4452; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:164453; CHECK-NEXT:    s_waitcnt vmcnt(6)4454; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:204455; CHECK-NEXT:    s_waitcnt vmcnt(5)4456; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:244457; CHECK-NEXT:    s_waitcnt vmcnt(4)4458; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:284459; CHECK-NEXT:    s_waitcnt vmcnt(3)4460; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen4461; CHECK-NEXT:    s_waitcnt vmcnt(2)4462; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:44463; CHECK-NEXT:    s_waitcnt vmcnt(1)4464; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:84465; CHECK-NEXT:    s_waitcnt vmcnt(0)4466; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124467; CHECK-NEXT:    s_setpc_b64 s[30:31]4468entry:4469  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 8 %dst, ptr addrspace(5) noundef nonnull align 8 %src, i64 32, i1 false)4470  ret void4471}4472 4473define void @memcpy_p5_p5_sz16_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {4474; CHECK-LABEL: memcpy_p5_p5_sz16_align_16_16:4475; CHECK:       ; %bb.0: ; %entry4476; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4477; CHECK-NEXT:    s_clause 0x34478; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen4479; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:44480; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:84481; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:124482; CHECK-NEXT:    s_waitcnt vmcnt(3)4483; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen4484; CHECK-NEXT:    s_waitcnt vmcnt(2)4485; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:44486; CHECK-NEXT:    s_waitcnt vmcnt(1)4487; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:84488; CHECK-NEXT:    s_waitcnt vmcnt(0)4489; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124490; CHECK-NEXT:    s_setpc_b64 s[30:31]4491entry:4492  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 16, i1 false)4493  ret void4494}4495 4496define void @memcpy_p5_p5_sz31_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {4497; CHECK-LABEL: memcpy_p5_p5_sz31_align_16_16:4498; CHECK:       ; %bb.0: ; %entry4499; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4500; CHECK-NEXT:    s_clause 0x74501; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:154502; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:194503; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:234504; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:274505; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen4506; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:44507; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:84508; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:124509; CHECK-NEXT:    s_waitcnt vmcnt(7)4510; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:154511; CHECK-NEXT:    s_waitcnt vmcnt(6)4512; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:194513; CHECK-NEXT:    s_waitcnt vmcnt(5)4514; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:234515; CHECK-NEXT:    s_waitcnt vmcnt(4)4516; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:274517; CHECK-NEXT:    s_waitcnt vmcnt(3)4518; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen4519; CHECK-NEXT:    s_waitcnt vmcnt(2)4520; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:44521; CHECK-NEXT:    s_waitcnt vmcnt(1)4522; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:84523; CHECK-NEXT:    s_waitcnt vmcnt(0)4524; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124525; CHECK-NEXT:    s_setpc_b64 s[30:31]4526entry:4527  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 31, i1 false)4528  ret void4529}4530 4531define void @memcpy_p5_p5_sz32_align_16_16(ptr addrspace(5) align 16 %dst, ptr addrspace(5) align 16 readonly %src) {4532; CHECK-LABEL: memcpy_p5_p5_sz32_align_16_16:4533; CHECK:       ; %bb.0: ; %entry4534; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4535; CHECK-NEXT:    s_clause 0x74536; CHECK-NEXT:    buffer_load_dword v2, v1, s[0:3], 0 offen offset:164537; CHECK-NEXT:    buffer_load_dword v3, v1, s[0:3], 0 offen offset:204538; CHECK-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen offset:244539; CHECK-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen offset:284540; CHECK-NEXT:    buffer_load_dword v6, v1, s[0:3], 0 offen4541; CHECK-NEXT:    buffer_load_dword v7, v1, s[0:3], 0 offen offset:44542; CHECK-NEXT:    buffer_load_dword v8, v1, s[0:3], 0 offen offset:84543; CHECK-NEXT:    buffer_load_dword v1, v1, s[0:3], 0 offen offset:124544; CHECK-NEXT:    s_waitcnt vmcnt(7)4545; CHECK-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:164546; CHECK-NEXT:    s_waitcnt vmcnt(6)4547; CHECK-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:204548; CHECK-NEXT:    s_waitcnt vmcnt(5)4549; CHECK-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:244550; CHECK-NEXT:    s_waitcnt vmcnt(4)4551; CHECK-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:284552; CHECK-NEXT:    s_waitcnt vmcnt(3)4553; CHECK-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen4554; CHECK-NEXT:    s_waitcnt vmcnt(2)4555; CHECK-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:44556; CHECK-NEXT:    s_waitcnt vmcnt(1)4557; CHECK-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:84558; CHECK-NEXT:    s_waitcnt vmcnt(0)4559; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:124560; CHECK-NEXT:    s_setpc_b64 s[30:31]4561entry:4562  tail call void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noundef nonnull align 16 %dst, ptr addrspace(5) noundef nonnull align 16 %src, i64 32, i1 false)4563  ret void4564}4565 4566declare void @llvm.memcpy.p0.p0.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(0) noalias nocapture readonly, i64, i1 immarg) #24567declare void @llvm.memcpy.p0.p1.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) #24568declare void @llvm.memcpy.p0.p3.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(3) noalias nocapture readonly, i64, i1 immarg) #24569declare void @llvm.memcpy.p0.p4.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #24570declare void @llvm.memcpy.p0.p5.i64(ptr addrspace(0) noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #24571declare void @llvm.memcpy.p1.p0.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(0) noalias nocapture readonly, i64, i1 immarg) #24572declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) #24573declare void @llvm.memcpy.p1.p3.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(3) noalias nocapture readonly, i64, i1 immarg) #24574declare void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #24575declare void @llvm.memcpy.p1.p5.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #24576declare void @llvm.memcpy.p3.p0.i64(ptr addrspace(3) noalias nocapture writeonly, ptr addrspace(0) noalias nocapture readonly, i64, i1 immarg) #24577declare void @llvm.memcpy.p3.p1.i64(ptr addrspace(3) noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) #24578declare void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noalias nocapture writeonly, ptr addrspace(3) noalias nocapture readonly, i64, i1 immarg) #24579declare void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #24580declare void @llvm.memcpy.p3.p5.i64(ptr addrspace(3) noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #24581declare void @llvm.memcpy.p5.p0.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(0) noalias nocapture readonly, i64, i1 immarg) #24582declare void @llvm.memcpy.p5.p1.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) #24583declare void @llvm.memcpy.p5.p3.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(3) noalias nocapture readonly, i64, i1 immarg) #24584declare void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #24585declare void @llvm.memcpy.p5.p5.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #24586 4587attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }4588 4589