2328 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 %s -o - | FileCheck %s3 4; Check code generation for memmoves with statically unknown size and all5; combinations of the following address spaces:6; destination address space: 0, 1, 3, 57; source address space: 0, 1, 3, 4, 58 9define void @memmove_p0_p0(ptr addrspace(0) align 1 %dst, ptr addrspace(0) align 1 readonly %src, i64 %sz) {10; CHECK-LABEL: memmove_p0_p0:11; CHECK: ; %bb.0: ; %entry12; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13; CHECK-NEXT: v_and_b32_e32 v6, 15, v414; CHECK-NEXT: v_mov_b32_e32 v7, 015; CHECK-NEXT: v_and_b32_e32 v8, -16, v416; CHECK-NEXT: v_mov_b32_e32 v9, v517; CHECK-NEXT: s_mov_b32 s6, exec_lo18; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]19; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]20; CHECK-NEXT: v_cmpx_ge_u64_e64 v[2:3], v[0:1]21; CHECK-NEXT: s_xor_b32 s7, exec_lo, s622; CHECK-NEXT: s_cbranch_execnz .LBB0_323; CHECK-NEXT: ; %bb.1: ; %Flow3524; CHECK-NEXT: s_andn2_saveexec_b32 s6, s725; CHECK-NEXT: s_cbranch_execnz .LBB0_1026; CHECK-NEXT: .LBB0_2: ; %Flow3627; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s628; CHECK-NEXT: s_waitcnt lgkmcnt(0)29; CHECK-NEXT: s_setpc_b64 s[30:31]30; CHECK-NEXT: .LBB0_3: ; %memmove_copy_forward31; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo32; CHECK-NEXT: s_cbranch_execz .LBB0_633; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader34; CHECK-NEXT: v_mov_b32_e32 v5, v335; CHECK-NEXT: v_mov_b32_e32 v11, v136; CHECK-NEXT: v_mov_b32_e32 v13, v937; CHECK-NEXT: v_mov_b32_e32 v4, v238; CHECK-NEXT: v_mov_b32_e32 v10, v039; CHECK-NEXT: v_mov_b32_e32 v12, v840; CHECK-NEXT: s_mov_b32 s9, 041; CHECK-NEXT: .p2align 642; CHECK-NEXT: .LBB0_5: ; %memmove_fwd_main_loop43; CHECK-NEXT: ; =>This Inner Loop Header: Depth=144; CHECK-NEXT: flat_load_dwordx4 v[14:17], v[4:5]45; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -1646; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s547; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 1648; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s549; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]50; CHECK-NEXT: s_or_b32 s9, s5, s951; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)52; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[14:17]53; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 1654; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s655; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s956; CHECK-NEXT: s_cbranch_execnz .LBB0_557; CHECK-NEXT: .LBB0_6: ; %Flow3058; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s859; CHECK-NEXT: s_and_saveexec_b32 s8, s460; CHECK-NEXT: s_cbranch_execz .LBB0_961; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader62; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v863; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s564; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v865; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s566; CHECK-NEXT: s_mov_b32 s9, 067; CHECK-NEXT: .p2align 668; CHECK-NEXT: .LBB0_8: ; %memmove_fwd_residual_loop69; CHECK-NEXT: ; =>This Inner Loop Header: Depth=170; CHECK-NEXT: flat_load_ubyte v4, v[2:3]71; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -172; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s573; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 174; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s575; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]76; CHECK-NEXT: s_or_b32 s9, s5, s977; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)78; CHECK-NEXT: flat_store_byte v[0:1], v479; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 180; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s681; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s982; CHECK-NEXT: s_cbranch_execnz .LBB0_883; CHECK-NEXT: .LBB0_9: ; %Flow2884; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s885; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr986; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr187; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr388; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr789; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr590; CHECK-NEXT: s_andn2_saveexec_b32 s6, s791; CHECK-NEXT: s_cbranch_execz .LBB0_292; CHECK-NEXT: .LBB0_10: ; %memmove_copy_backwards93; CHECK-NEXT: s_and_saveexec_b32 s7, s494; CHECK-NEXT: s_cbranch_execz .LBB0_1395; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader96; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -197; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s498; CHECK-NEXT: s_mov_b32 s8, 099; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10100; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4101; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10102; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4103; CHECK-NEXT: .p2align 6104; CHECK-NEXT: .LBB0_12: ; %memmove_bwd_residual_loop105; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1106; CHECK-NEXT: flat_load_ubyte v12, v[10:11]107; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1108; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4109; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1110; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4111; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]112; CHECK-NEXT: s_or_b32 s8, s4, s8113; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)114; CHECK-NEXT: flat_store_byte v[4:5], v12115; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1116; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5117; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8118; CHECK-NEXT: s_cbranch_execnz .LBB0_12119; CHECK-NEXT: .LBB0_13: ; %Flow34120; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7121; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo122; CHECK-NEXT: s_cbranch_execz .LBB0_16123; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader124; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16125; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo126; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, -16127; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo128; CHECK-NEXT: s_mov_b32 s7, 0129; CHECK-NEXT: .p2align 6130; CHECK-NEXT: .LBB0_15: ; %memmove_bwd_main_loop131; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1132; CHECK-NEXT: v_mov_b32_e32 v11, v9133; CHECK-NEXT: v_mov_b32_e32 v10, v8134; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10135; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo136; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16137; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo138; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[4:5]139; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10140; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]141; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4142; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7143; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)144; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[4:7]145; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7146; CHECK-NEXT: s_cbranch_execnz .LBB0_15147; CHECK-NEXT: .LBB0_16: ; %Flow32148; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s5149; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6150; CHECK-NEXT: s_waitcnt lgkmcnt(0)151; CHECK-NEXT: s_setpc_b64 s[30:31]152entry:153 tail call void @llvm.memmove.p0.p0.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 %sz, i1 false)154 ret void155}156 157define void @memmove_p0_p1(ptr addrspace(0) align 1 %dst, ptr addrspace(1) align 1 readonly %src, i64 %sz) {158; CHECK-LABEL: memmove_p0_p1:159; CHECK: ; %bb.0: ; %entry160; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)161; CHECK-NEXT: v_and_b32_e32 v6, 15, v4162; CHECK-NEXT: v_mov_b32_e32 v7, 0163; CHECK-NEXT: v_and_b32_e32 v8, -16, v4164; CHECK-NEXT: v_mov_b32_e32 v9, v5165; CHECK-NEXT: s_mov_b32 s6, exec_lo166; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]167; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]168; CHECK-NEXT: v_cmpx_ge_u64_e64 v[2:3], v[0:1]169; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6170; CHECK-NEXT: s_cbranch_execnz .LBB1_3171; CHECK-NEXT: ; %bb.1: ; %Flow37172; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7173; CHECK-NEXT: s_cbranch_execnz .LBB1_10174; CHECK-NEXT: .LBB1_2: ; %Flow38175; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6176; CHECK-NEXT: s_waitcnt lgkmcnt(0)177; CHECK-NEXT: s_setpc_b64 s[30:31]178; CHECK-NEXT: .LBB1_3: ; %memmove_copy_forward179; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo180; CHECK-NEXT: s_cbranch_execz .LBB1_6181; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader182; CHECK-NEXT: v_mov_b32_e32 v5, v3183; CHECK-NEXT: v_mov_b32_e32 v11, v1184; CHECK-NEXT: v_mov_b32_e32 v13, v9185; CHECK-NEXT: v_mov_b32_e32 v4, v2186; CHECK-NEXT: v_mov_b32_e32 v10, v0187; CHECK-NEXT: v_mov_b32_e32 v12, v8188; CHECK-NEXT: s_mov_b32 s9, 0189; CHECK-NEXT: .p2align 6190; CHECK-NEXT: .LBB1_5: ; %memmove_fwd_main_loop191; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1192; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off193; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16194; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5195; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16196; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5197; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]198; CHECK-NEXT: s_or_b32 s9, s5, s9199; CHECK-NEXT: s_waitcnt vmcnt(0)200; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[14:17]201; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16202; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6203; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9204; CHECK-NEXT: s_cbranch_execnz .LBB1_5205; CHECK-NEXT: .LBB1_6: ; %Flow32206; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8207; CHECK-NEXT: s_and_saveexec_b32 s8, s4208; CHECK-NEXT: s_cbranch_execz .LBB1_9209; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader210; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8211; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5212; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8213; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5214; CHECK-NEXT: s_mov_b32 s9, 0215; CHECK-NEXT: .p2align 6216; CHECK-NEXT: .LBB1_8: ; %memmove_fwd_residual_loop217; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1218; CHECK-NEXT: global_load_ubyte v4, v[2:3], off219; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1220; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5221; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1222; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5223; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]224; CHECK-NEXT: s_or_b32 s9, s5, s9225; CHECK-NEXT: s_waitcnt vmcnt(0)226; CHECK-NEXT: flat_store_byte v[0:1], v4227; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 1228; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s6229; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9230; CHECK-NEXT: s_cbranch_execnz .LBB1_8231; CHECK-NEXT: .LBB1_9: ; %Flow30232; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8233; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9234; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3235; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1236; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7237; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5238; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7239; CHECK-NEXT: s_cbranch_execz .LBB1_2240; CHECK-NEXT: .LBB1_10: ; %memmove_copy_backwards241; CHECK-NEXT: s_and_saveexec_b32 s7, s4242; CHECK-NEXT: s_cbranch_execz .LBB1_13243; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader244; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1245; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4246; CHECK-NEXT: s_mov_b32 s8, 0247; CHECK-NEXT: v_add_co_u32 v4, s4, v2, v10248; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, s4249; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10250; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4251; CHECK-NEXT: .p2align 6252; CHECK-NEXT: .LBB1_12: ; %memmove_bwd_residual_loop253; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1254; CHECK-NEXT: global_load_ubyte v12, v[4:5], off255; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1256; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4257; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -1258; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s4259; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]260; CHECK-NEXT: s_or_b32 s8, s4, s8261; CHECK-NEXT: s_waitcnt vmcnt(0)262; CHECK-NEXT: flat_store_byte v[10:11], v12263; CHECK-NEXT: v_add_co_u32 v10, s5, v10, -1264; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s5265; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8266; CHECK-NEXT: s_cbranch_execnz .LBB1_12267; CHECK-NEXT: .LBB1_13: ; %Flow36268; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7269; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo270; CHECK-NEXT: s_cbranch_execz .LBB1_16271; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader272; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, -16273; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo274; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16275; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo276; CHECK-NEXT: s_mov_b32 s7, 0277; CHECK-NEXT: .p2align 6278; CHECK-NEXT: .LBB1_15: ; %memmove_bwd_main_loop279; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1280; CHECK-NEXT: v_mov_b32_e32 v11, v9281; CHECK-NEXT: v_mov_b32_e32 v10, v8282; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10283; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo284; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16285; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo286; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off287; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10288; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]289; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4290; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7291; CHECK-NEXT: s_waitcnt vmcnt(0)292; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[4:7]293; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7294; CHECK-NEXT: s_cbranch_execnz .LBB1_15295; CHECK-NEXT: .LBB1_16: ; %Flow34296; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s5297; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6298; CHECK-NEXT: s_waitcnt lgkmcnt(0)299; CHECK-NEXT: s_setpc_b64 s[30:31]300entry:301 tail call void @llvm.memmove.p0.p1.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 %sz, i1 false)302 ret void303}304 305define void @memmove_p0_p3(ptr addrspace(0) align 1 %dst, ptr addrspace(3) align 1 readonly %src, i64 %sz) {306; CHECK-LABEL: memmove_p0_p3:307; CHECK: ; %bb.0: ; %entry308; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)309; CHECK-NEXT: v_and_b32_e32 v7, 15, v3310; CHECK-NEXT: v_mov_b32_e32 v8, 0311; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]312; CHECK-NEXT: v_and_b32_e32 v5, -16, v3313; CHECK-NEXT: v_mov_b32_e32 v6, v4314; CHECK-NEXT: s_mov_b32 s6, exec_lo315; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]316; CHECK-NEXT: v_cndmask_b32_e32 v9, -1, v0, vcc_lo317; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[5:6]318; CHECK-NEXT: v_cmpx_ge_u32_e64 v2, v9319; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6320; CHECK-NEXT: s_cbranch_execnz .LBB2_3321; CHECK-NEXT: ; %bb.1: ; %Flow39322; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7323; CHECK-NEXT: s_cbranch_execnz .LBB2_10324; CHECK-NEXT: .LBB2_2: ; %Flow40325; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6326; CHECK-NEXT: s_waitcnt lgkmcnt(0)327; CHECK-NEXT: s_setpc_b64 s[30:31]328; CHECK-NEXT: .LBB2_3: ; %memmove_copy_forward329; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo330; CHECK-NEXT: s_cbranch_execz .LBB2_6331; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader332; CHECK-NEXT: v_mov_b32_e32 v10, v1333; CHECK-NEXT: v_mov_b32_e32 v12, v6334; CHECK-NEXT: v_mov_b32_e32 v9, v0335; CHECK-NEXT: v_mov_b32_e32 v11, v5336; CHECK-NEXT: v_mov_b32_e32 v4, v2337; CHECK-NEXT: s_mov_b32 s9, 0338; CHECK-NEXT: .p2align 6339; CHECK-NEXT: .LBB2_5: ; %memmove_fwd_main_loop340; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1341; CHECK-NEXT: ds_read_b128 v[13:16], v4342; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -16343; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s5344; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v4345; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[11:12]346; CHECK-NEXT: s_or_b32 s9, s5, s9347; CHECK-NEXT: s_waitcnt lgkmcnt(0)348; CHECK-NEXT: flat_store_dwordx4 v[9:10], v[13:16]349; CHECK-NEXT: v_add_co_u32 v9, s6, v9, 16350; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s6351; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9352; CHECK-NEXT: s_cbranch_execnz .LBB2_5353; CHECK-NEXT: .LBB2_6: ; %Flow34354; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8355; CHECK-NEXT: s_and_saveexec_b32 s8, s4356; CHECK-NEXT: s_cbranch_execz .LBB2_9357; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader358; CHECK-NEXT: v_and_b32_e32 v3, -16, v3359; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v5360; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v6, s5361; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3362; CHECK-NEXT: s_mov_b32 s9, 0363; CHECK-NEXT: .p2align 6364; CHECK-NEXT: .LBB2_8: ; %memmove_fwd_residual_loop365; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1366; CHECK-NEXT: ds_read_u8 v3, v2367; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1368; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5369; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2370; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]371; CHECK-NEXT: s_or_b32 s9, s5, s9372; CHECK-NEXT: s_waitcnt lgkmcnt(0)373; CHECK-NEXT: flat_store_byte v[0:1], v3374; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 1375; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s6376; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9377; CHECK-NEXT: s_cbranch_execnz .LBB2_8378; CHECK-NEXT: .LBB2_9: ; %Flow32379; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8380; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6381; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr4382; CHECK-NEXT: ; implicit-def: $vgpr2383; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1384; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8385; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7386; CHECK-NEXT: s_cbranch_execz .LBB2_2387; CHECK-NEXT: .LBB2_10: ; %memmove_copy_backwards388; CHECK-NEXT: s_and_saveexec_b32 s7, s4389; CHECK-NEXT: s_cbranch_execz .LBB2_13390; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader391; CHECK-NEXT: v_add_co_u32 v9, s4, v0, v3392; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v1, v4, s4393; CHECK-NEXT: v_add3_u32 v4, v3, v2, -1394; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1395; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4396; CHECK-NEXT: s_mov_b32 s8, 0397; CHECK-NEXT: .p2align 6398; CHECK-NEXT: .LBB2_12: ; %memmove_bwd_residual_loop399; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1400; CHECK-NEXT: ds_read_u8 v11, v4401; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1402; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4403; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4404; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]405; CHECK-NEXT: s_or_b32 s8, s4, s8406; CHECK-NEXT: s_waitcnt lgkmcnt(0)407; CHECK-NEXT: flat_store_byte v[9:10], v11408; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -1409; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5410; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8411; CHECK-NEXT: s_cbranch_execnz .LBB2_12412; CHECK-NEXT: .LBB2_13: ; %Flow38413; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7414; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo415; CHECK-NEXT: s_cbranch_execz .LBB2_16416; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader417; CHECK-NEXT: v_and_b32_e32 v3, -16, v3418; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16419; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo420; CHECK-NEXT: v_add3_u32 v2, v3, v2, -16421; CHECK-NEXT: s_mov_b32 s7, 0422; CHECK-NEXT: .p2align 6423; CHECK-NEXT: .LBB2_15: ; %memmove_bwd_main_loop424; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1425; CHECK-NEXT: ds_read_b128 v[7:10], v2426; CHECK-NEXT: v_mov_b32_e32 v3, v5427; CHECK-NEXT: v_mov_b32_e32 v4, v6428; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2429; CHECK-NEXT: v_add_co_u32 v5, vcc_lo, v3, -16430; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v4, vcc_lo431; CHECK-NEXT: v_add_co_u32 v3, s4, v0, v3432; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v1, v4, s4433; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[5:6]434; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7435; CHECK-NEXT: s_waitcnt lgkmcnt(0)436; CHECK-NEXT: flat_store_dwordx4 v[3:4], v[7:10]437; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7438; CHECK-NEXT: s_cbranch_execnz .LBB2_15439; CHECK-NEXT: .LBB2_16: ; %Flow36440; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s5441; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6442; CHECK-NEXT: s_waitcnt lgkmcnt(0)443; CHECK-NEXT: s_setpc_b64 s[30:31]444entry:445 tail call void @llvm.memmove.p0.p3.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 %sz, i1 false)446 ret void447}448 449define void @memmove_p0_p4(ptr addrspace(0) align 1 %dst, ptr addrspace(4) align 1 readonly %src, i64 %sz) {450; CHECK-LABEL: memmove_p0_p4:451; CHECK: ; %bb.0: ; %entry452; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)453; CHECK-NEXT: v_and_b32_e32 v6, 15, v4454; CHECK-NEXT: v_mov_b32_e32 v7, 0455; CHECK-NEXT: v_and_b32_e32 v8, -16, v4456; CHECK-NEXT: v_mov_b32_e32 v9, v5457; CHECK-NEXT: s_mov_b32 s6, exec_lo458; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]459; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]460; CHECK-NEXT: v_cmpx_ge_u64_e64 v[2:3], v[0:1]461; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6462; CHECK-NEXT: s_cbranch_execnz .LBB3_3463; CHECK-NEXT: ; %bb.1: ; %Flow36464; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7465; CHECK-NEXT: s_cbranch_execnz .LBB3_10466; CHECK-NEXT: .LBB3_2: ; %Flow37467; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6468; CHECK-NEXT: s_waitcnt lgkmcnt(0)469; CHECK-NEXT: s_setpc_b64 s[30:31]470; CHECK-NEXT: .LBB3_3: ; %memmove_copy_forward471; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo472; CHECK-NEXT: s_cbranch_execz .LBB3_6473; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader474; CHECK-NEXT: v_mov_b32_e32 v5, v3475; CHECK-NEXT: v_mov_b32_e32 v11, v1476; CHECK-NEXT: v_mov_b32_e32 v13, v9477; CHECK-NEXT: v_mov_b32_e32 v4, v2478; CHECK-NEXT: v_mov_b32_e32 v10, v0479; CHECK-NEXT: v_mov_b32_e32 v12, v8480; CHECK-NEXT: s_mov_b32 s9, 0481; CHECK-NEXT: .p2align 6482; CHECK-NEXT: .LBB3_5: ; %memmove_fwd_main_loop483; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1484; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off485; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16486; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5487; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16488; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5489; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]490; CHECK-NEXT: s_or_b32 s9, s5, s9491; CHECK-NEXT: s_waitcnt vmcnt(0)492; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[14:17]493; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16494; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6495; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9496; CHECK-NEXT: s_cbranch_execnz .LBB3_5497; CHECK-NEXT: .LBB3_6: ; %Flow31498; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8499; CHECK-NEXT: s_and_saveexec_b32 s8, s4500; CHECK-NEXT: s_cbranch_execz .LBB3_9501; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader502; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8503; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5504; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8505; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5506; CHECK-NEXT: s_mov_b32 s9, 0507; CHECK-NEXT: .p2align 6508; CHECK-NEXT: .LBB3_8: ; %memmove_fwd_residual_loop509; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1510; CHECK-NEXT: global_load_ubyte v4, v[2:3], off511; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1512; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5513; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1514; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5515; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]516; CHECK-NEXT: s_or_b32 s9, s5, s9517; CHECK-NEXT: s_waitcnt vmcnt(0)518; CHECK-NEXT: flat_store_byte v[0:1], v4519; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 1520; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s6521; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9522; CHECK-NEXT: s_cbranch_execnz .LBB3_8523; CHECK-NEXT: .LBB3_9: ; %Flow29524; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8525; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9526; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3527; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1528; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7529; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5530; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7531; CHECK-NEXT: s_cbranch_execz .LBB3_2532; CHECK-NEXT: .LBB3_10: ; %memmove_copy_backwards533; CHECK-NEXT: s_and_saveexec_b32 s7, s4534; CHECK-NEXT: s_cbranch_execz .LBB3_13535; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader536; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1537; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4538; CHECK-NEXT: s_mov_b32 s8, 0539; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10540; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4541; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10542; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4543; CHECK-NEXT: .p2align 6544; CHECK-NEXT: .LBB3_12: ; %memmove_bwd_residual_loop545; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1546; CHECK-NEXT: global_load_ubyte v12, v[10:11], off547; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1548; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4549; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1550; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4551; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]552; CHECK-NEXT: s_or_b32 s8, s4, s8553; CHECK-NEXT: s_waitcnt vmcnt(0)554; CHECK-NEXT: flat_store_byte v[4:5], v12555; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1556; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5557; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8558; CHECK-NEXT: s_cbranch_execnz .LBB3_12559; CHECK-NEXT: .LBB3_13: ; %Flow35560; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7561; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo562; CHECK-NEXT: s_cbranch_execz .LBB3_16563; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader564; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, -16565; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo566; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16567; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo568; CHECK-NEXT: s_mov_b32 s7, 0569; CHECK-NEXT: .p2align 6570; CHECK-NEXT: .LBB3_15: ; %memmove_bwd_main_loop571; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1572; CHECK-NEXT: v_mov_b32_e32 v11, v9573; CHECK-NEXT: v_mov_b32_e32 v10, v8574; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10575; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo576; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16577; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo578; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off579; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10580; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]581; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4582; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7583; CHECK-NEXT: s_waitcnt vmcnt(0)584; CHECK-NEXT: flat_store_dwordx4 v[10:11], v[4:7]585; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7586; CHECK-NEXT: s_cbranch_execnz .LBB3_15587; CHECK-NEXT: .LBB3_16: ; %Flow33588; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s5589; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6590; CHECK-NEXT: s_waitcnt lgkmcnt(0)591; CHECK-NEXT: s_setpc_b64 s[30:31]592entry:593 tail call void @llvm.memmove.p0.p4.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 %sz, i1 false)594 ret void595}596 597define void @memmove_p0_p5(ptr addrspace(0) align 1 %dst, ptr addrspace(5) align 1 readonly %src, i64 %sz) {598; CHECK-LABEL: memmove_p0_p5:599; CHECK: ; %bb.0: ; %entry600; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)601; CHECK-NEXT: v_and_b32_e32 v7, 15, v3602; CHECK-NEXT: v_mov_b32_e32 v8, 0603; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]604; CHECK-NEXT: v_and_b32_e32 v5, -16, v3605; CHECK-NEXT: v_mov_b32_e32 v6, v4606; CHECK-NEXT: s_mov_b32 s6, exec_lo607; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[7:8]608; CHECK-NEXT: v_cndmask_b32_e32 v9, -1, v0, vcc_lo609; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[5:6]610; CHECK-NEXT: v_cmpx_ge_u32_e64 v2, v9611; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6612; CHECK-NEXT: s_cbranch_execnz .LBB4_3613; CHECK-NEXT: ; %bb.1: ; %Flow39614; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7615; CHECK-NEXT: s_cbranch_execnz .LBB4_10616; CHECK-NEXT: .LBB4_2: ; %Flow40617; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6618; CHECK-NEXT: s_waitcnt lgkmcnt(0)619; CHECK-NEXT: s_setpc_b64 s[30:31]620; CHECK-NEXT: .LBB4_3: ; %memmove_copy_forward621; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo622; CHECK-NEXT: s_cbranch_execz .LBB4_6623; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader624; CHECK-NEXT: v_mov_b32_e32 v10, v1625; CHECK-NEXT: v_mov_b32_e32 v12, v6626; CHECK-NEXT: v_mov_b32_e32 v9, v0627; CHECK-NEXT: v_mov_b32_e32 v11, v5628; CHECK-NEXT: v_mov_b32_e32 v4, v2629; CHECK-NEXT: s_mov_b32 s9, 0630; CHECK-NEXT: .p2align 6631; CHECK-NEXT: .LBB4_5: ; %memmove_fwd_main_loop632; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1633; CHECK-NEXT: s_clause 0x3634; CHECK-NEXT: buffer_load_dword v13, v4, s[0:3], 0 offen635; CHECK-NEXT: buffer_load_dword v14, v4, s[0:3], 0 offen offset:4636; CHECK-NEXT: buffer_load_dword v15, v4, s[0:3], 0 offen offset:8637; CHECK-NEXT: buffer_load_dword v16, v4, s[0:3], 0 offen offset:12638; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -16639; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s5640; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v4641; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[11:12]642; CHECK-NEXT: s_or_b32 s9, s5, s9643; CHECK-NEXT: s_waitcnt vmcnt(0)644; CHECK-NEXT: flat_store_dwordx4 v[9:10], v[13:16]645; CHECK-NEXT: v_add_co_u32 v9, s6, v9, 16646; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s6647; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9648; CHECK-NEXT: s_cbranch_execnz .LBB4_5649; CHECK-NEXT: .LBB4_6: ; %Flow34650; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8651; CHECK-NEXT: s_and_saveexec_b32 s8, s4652; CHECK-NEXT: s_cbranch_execz .LBB4_9653; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader654; CHECK-NEXT: v_and_b32_e32 v3, -16, v3655; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v5656; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v6, s5657; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v3658; CHECK-NEXT: s_mov_b32 s9, 0659; CHECK-NEXT: .p2align 6660; CHECK-NEXT: .LBB4_8: ; %memmove_fwd_residual_loop661; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1662; CHECK-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen663; CHECK-NEXT: v_add_co_u32 v7, s5, v7, -1664; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s5665; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v2666; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[7:8]667; CHECK-NEXT: s_or_b32 s9, s5, s9668; CHECK-NEXT: s_waitcnt vmcnt(0)669; CHECK-NEXT: flat_store_byte v[0:1], v3670; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 1671; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s6672; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9673; CHECK-NEXT: s_cbranch_execnz .LBB4_8674; CHECK-NEXT: .LBB4_9: ; %Flow32675; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8676; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr6677; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr4678; CHECK-NEXT: ; implicit-def: $vgpr2679; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1680; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr8681; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7682; CHECK-NEXT: s_cbranch_execz .LBB4_2683; CHECK-NEXT: .LBB4_10: ; %memmove_copy_backwards684; CHECK-NEXT: s_and_saveexec_b32 s7, s4685; CHECK-NEXT: s_cbranch_execz .LBB4_13686; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader687; CHECK-NEXT: v_add_co_u32 v9, s4, v0, v3688; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v1, v4, s4689; CHECK-NEXT: v_add3_u32 v4, v3, v2, -1690; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -1691; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s4692; CHECK-NEXT: s_mov_b32 s8, 0693; CHECK-NEXT: .p2align 6694; CHECK-NEXT: .LBB4_12: ; %memmove_bwd_residual_loop695; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1696; CHECK-NEXT: buffer_load_ubyte v11, v4, s[0:3], 0 offen697; CHECK-NEXT: v_add_co_u32 v7, s4, v7, -1698; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, s4699; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v4700; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[7:8]701; CHECK-NEXT: s_or_b32 s8, s4, s8702; CHECK-NEXT: s_waitcnt vmcnt(0)703; CHECK-NEXT: flat_store_byte v[9:10], v11704; CHECK-NEXT: v_add_co_u32 v9, s5, v9, -1705; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s5706; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8707; CHECK-NEXT: s_cbranch_execnz .LBB4_12708; CHECK-NEXT: .LBB4_13: ; %Flow38709; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7710; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo711; CHECK-NEXT: s_cbranch_execz .LBB4_16712; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader713; CHECK-NEXT: v_and_b32_e32 v3, -16, v3714; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16715; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo716; CHECK-NEXT: v_add3_u32 v2, v3, v2, -16717; CHECK-NEXT: s_mov_b32 s7, 0718; CHECK-NEXT: .p2align 6719; CHECK-NEXT: .LBB4_15: ; %memmove_bwd_main_loop720; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1721; CHECK-NEXT: s_clause 0x3722; CHECK-NEXT: buffer_load_dword v7, v2, s[0:3], 0 offen723; CHECK-NEXT: buffer_load_dword v8, v2, s[0:3], 0 offen offset:4724; CHECK-NEXT: buffer_load_dword v9, v2, s[0:3], 0 offen offset:8725; CHECK-NEXT: buffer_load_dword v10, v2, s[0:3], 0 offen offset:12726; CHECK-NEXT: v_mov_b32_e32 v3, v5727; CHECK-NEXT: v_mov_b32_e32 v4, v6728; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v2729; CHECK-NEXT: v_add_co_u32 v5, vcc_lo, v3, -16730; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v4, vcc_lo731; CHECK-NEXT: v_add_co_u32 v3, s4, v0, v3732; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v1, v4, s4733; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[5:6]734; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7735; CHECK-NEXT: s_waitcnt vmcnt(0)736; CHECK-NEXT: flat_store_dwordx4 v[3:4], v[7:10]737; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7738; CHECK-NEXT: s_cbranch_execnz .LBB4_15739; CHECK-NEXT: .LBB4_16: ; %Flow36740; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s5741; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6742; CHECK-NEXT: s_waitcnt lgkmcnt(0)743; CHECK-NEXT: s_setpc_b64 s[30:31]744entry:745 tail call void @llvm.memmove.p0.p5.i64(ptr addrspace(0) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 %sz, i1 false)746 ret void747}748 749 750define void @memmove_p1_p0(ptr addrspace(1) align 1 %dst, ptr addrspace(0) align 1 readonly %src, i64 %sz) {751; CHECK-LABEL: memmove_p1_p0:752; CHECK: ; %bb.0: ; %entry753; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)754; CHECK-NEXT: v_and_b32_e32 v6, 15, v4755; CHECK-NEXT: v_mov_b32_e32 v7, 0756; CHECK-NEXT: v_and_b32_e32 v8, -16, v4757; CHECK-NEXT: v_mov_b32_e32 v9, v5758; CHECK-NEXT: s_mov_b32 s6, exec_lo759; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]760; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]761; CHECK-NEXT: v_cmpx_ge_u64_e64 v[2:3], v[0:1]762; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6763; CHECK-NEXT: s_cbranch_execnz .LBB5_3764; CHECK-NEXT: ; %bb.1: ; %Flow37765; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7766; CHECK-NEXT: s_cbranch_execnz .LBB5_10767; CHECK-NEXT: .LBB5_2: ; %Flow38768; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6769; CHECK-NEXT: s_setpc_b64 s[30:31]770; CHECK-NEXT: .LBB5_3: ; %memmove_copy_forward771; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo772; CHECK-NEXT: s_cbranch_execz .LBB5_6773; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader774; CHECK-NEXT: v_mov_b32_e32 v5, v3775; CHECK-NEXT: v_mov_b32_e32 v11, v1776; CHECK-NEXT: v_mov_b32_e32 v13, v9777; CHECK-NEXT: v_mov_b32_e32 v4, v2778; CHECK-NEXT: v_mov_b32_e32 v10, v0779; CHECK-NEXT: v_mov_b32_e32 v12, v8780; CHECK-NEXT: s_mov_b32 s9, 0781; CHECK-NEXT: .p2align 6782; CHECK-NEXT: .LBB5_5: ; %memmove_fwd_main_loop783; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1784; CHECK-NEXT: flat_load_dwordx4 v[14:17], v[4:5]785; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16786; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5787; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16788; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5789; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]790; CHECK-NEXT: s_or_b32 s9, s5, s9791; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)792; CHECK-NEXT: global_store_dwordx4 v[10:11], v[14:17], off793; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16794; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6795; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9796; CHECK-NEXT: s_cbranch_execnz .LBB5_5797; CHECK-NEXT: .LBB5_6: ; %Flow32798; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8799; CHECK-NEXT: s_and_saveexec_b32 s8, s4800; CHECK-NEXT: s_cbranch_execz .LBB5_9801; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader802; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8803; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5804; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8805; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5806; CHECK-NEXT: s_mov_b32 s9, 0807; CHECK-NEXT: .p2align 6808; CHECK-NEXT: .LBB5_8: ; %memmove_fwd_residual_loop809; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1810; CHECK-NEXT: flat_load_ubyte v4, v[2:3]811; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1812; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5813; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1814; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5815; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]816; CHECK-NEXT: s_or_b32 s9, s5, s9817; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)818; CHECK-NEXT: global_store_byte v[0:1], v4, off819; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 1820; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s6821; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9822; CHECK-NEXT: s_cbranch_execnz .LBB5_8823; CHECK-NEXT: .LBB5_9: ; %Flow30824; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8825; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9826; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1827; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3828; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7829; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5830; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7831; CHECK-NEXT: s_cbranch_execz .LBB5_2832; CHECK-NEXT: .LBB5_10: ; %memmove_copy_backwards833; CHECK-NEXT: s_and_saveexec_b32 s7, s4834; CHECK-NEXT: s_cbranch_execz .LBB5_13835; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader836; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1837; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4838; CHECK-NEXT: s_mov_b32 s8, 0839; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10840; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4841; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10842; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4843; CHECK-NEXT: .p2align 6844; CHECK-NEXT: .LBB5_12: ; %memmove_bwd_residual_loop845; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1846; CHECK-NEXT: flat_load_ubyte v12, v[10:11]847; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1848; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4849; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1850; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4851; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]852; CHECK-NEXT: s_or_b32 s8, s4, s8853; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)854; CHECK-NEXT: global_store_byte v[4:5], v12, off855; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -1856; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s5857; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8858; CHECK-NEXT: s_cbranch_execnz .LBB5_12859; CHECK-NEXT: .LBB5_13: ; %Flow36860; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7861; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo862; CHECK-NEXT: s_cbranch_execz .LBB5_16863; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader864; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -16865; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo866; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, -16867; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo868; CHECK-NEXT: s_mov_b32 s7, 0869; CHECK-NEXT: .p2align 6870; CHECK-NEXT: .LBB5_15: ; %memmove_bwd_main_loop871; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1872; CHECK-NEXT: v_mov_b32_e32 v11, v9873; CHECK-NEXT: v_mov_b32_e32 v10, v8874; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v10875; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo876; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -16877; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo878; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[4:5]879; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v10880; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]881; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s4882; CHECK-NEXT: s_or_b32 s7, vcc_lo, s7883; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)884; CHECK-NEXT: global_store_dwordx4 v[10:11], v[4:7], off885; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s7886; CHECK-NEXT: s_cbranch_execnz .LBB5_15887; CHECK-NEXT: .LBB5_16: ; %Flow34888; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s5889; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6890; CHECK-NEXT: s_setpc_b64 s[30:31]891entry:892 tail call void @llvm.memmove.p1.p0.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 %sz, i1 false)893 ret void894}895 896define void @memmove_p1_p1(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align 1 readonly %src, i64 %sz) {897; CHECK-LABEL: memmove_p1_p1:898; CHECK: ; %bb.0: ; %entry899; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)900; CHECK-NEXT: v_and_b32_e32 v6, 15, v4901; CHECK-NEXT: v_mov_b32_e32 v7, 0902; CHECK-NEXT: v_and_b32_e32 v8, -16, v4903; CHECK-NEXT: v_mov_b32_e32 v9, v5904; CHECK-NEXT: s_mov_b32 s6, exec_lo905; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]906; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]907; CHECK-NEXT: v_cmpx_ge_u64_e64 v[2:3], v[0:1]908; CHECK-NEXT: s_xor_b32 s7, exec_lo, s6909; CHECK-NEXT: s_cbranch_execnz .LBB6_3910; CHECK-NEXT: ; %bb.1: ; %Flow39911; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7912; CHECK-NEXT: s_cbranch_execnz .LBB6_10913; CHECK-NEXT: .LBB6_2: ; %Flow40914; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s6915; CHECK-NEXT: s_setpc_b64 s[30:31]916; CHECK-NEXT: .LBB6_3: ; %memmove_copy_forward917; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo918; CHECK-NEXT: s_cbranch_execz .LBB6_6919; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader920; CHECK-NEXT: v_mov_b32_e32 v5, v3921; CHECK-NEXT: v_mov_b32_e32 v11, v1922; CHECK-NEXT: v_mov_b32_e32 v13, v9923; CHECK-NEXT: v_mov_b32_e32 v4, v2924; CHECK-NEXT: v_mov_b32_e32 v10, v0925; CHECK-NEXT: v_mov_b32_e32 v12, v8926; CHECK-NEXT: s_mov_b32 s9, 0927; CHECK-NEXT: .p2align 6928; CHECK-NEXT: .LBB6_5: ; %memmove_fwd_main_loop929; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1930; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off931; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -16932; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s5933; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 16934; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s5935; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]936; CHECK-NEXT: s_or_b32 s9, s5, s9937; CHECK-NEXT: s_waitcnt vmcnt(0)938; CHECK-NEXT: global_store_dwordx4 v[10:11], v[14:17], off939; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 16940; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s6941; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9942; CHECK-NEXT: s_cbranch_execnz .LBB6_5943; CHECK-NEXT: .LBB6_6: ; %Flow34944; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8945; CHECK-NEXT: s_and_saveexec_b32 s8, s4946; CHECK-NEXT: s_cbranch_execz .LBB6_9947; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader948; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v8949; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s5950; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v8951; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s5952; CHECK-NEXT: s_mov_b32 s9, 0953; CHECK-NEXT: .p2align 6954; CHECK-NEXT: .LBB6_8: ; %memmove_fwd_residual_loop955; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1956; CHECK-NEXT: global_load_ubyte v4, v[2:3], off957; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -1958; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s5959; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 1960; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s5961; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]962; CHECK-NEXT: s_or_b32 s9, s5, s9963; CHECK-NEXT: s_waitcnt vmcnt(0)964; CHECK-NEXT: global_store_byte v[0:1], v4, off965; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 1966; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s6967; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s9968; CHECK-NEXT: s_cbranch_execnz .LBB6_8969; CHECK-NEXT: .LBB6_9: ; %Flow32970; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s8971; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr9972; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr1973; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3974; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7975; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr5976; CHECK-NEXT: s_andn2_saveexec_b32 s6, s7977; CHECK-NEXT: s_cbranch_execz .LBB6_2978; CHECK-NEXT: .LBB6_10: ; %memmove_copy_backwards979; CHECK-NEXT: s_and_saveexec_b32 s7, s4980; CHECK-NEXT: s_cbranch_execz .LBB6_13981; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader982; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -1983; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s4984; CHECK-NEXT: s_mov_b32 s8, 0985; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v10986; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s4987; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v10988; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s4989; CHECK-NEXT: .p2align 6990; CHECK-NEXT: .LBB6_12: ; %memmove_bwd_residual_loop991; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1992; CHECK-NEXT: global_load_ubyte v12, v[10:11], off993; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -1994; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s4995; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -1996; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s4997; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]998; CHECK-NEXT: s_or_b32 s8, s4, s8999; CHECK-NEXT: s_waitcnt vmcnt(0)1000; CHECK-NEXT: global_store_byte v[4:5], v12, off1001; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -11002; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s51003; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s81004; CHECK-NEXT: s_cbranch_execnz .LBB6_121005; CHECK-NEXT: .LBB6_13: ; %Flow381006; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71007; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo1008; CHECK-NEXT: s_cbranch_execz .LBB6_161009; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader1010; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -161011; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo1012; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, -161013; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo1014; CHECK-NEXT: s_mov_b32 s7, 01015; CHECK-NEXT: .p2align 61016; CHECK-NEXT: .LBB6_15: ; %memmove_bwd_main_loop1017; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11018; CHECK-NEXT: v_mov_b32_e32 v11, v91019; CHECK-NEXT: v_mov_b32_e32 v10, v81020; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v101021; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo1022; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -161023; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo1024; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off1025; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v101026; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]1027; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s41028; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71029; CHECK-NEXT: s_waitcnt vmcnt(0)1030; CHECK-NEXT: global_store_dwordx4 v[10:11], v[4:7], off1031; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71032; CHECK-NEXT: s_cbranch_execnz .LBB6_151033; CHECK-NEXT: .LBB6_16: ; %Flow361034; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s51035; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61036; CHECK-NEXT: s_setpc_b64 s[30:31]1037entry:1038 tail call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 %sz, i1 false)1039 ret void1040}1041 1042define void @memmove_p1_p3(ptr addrspace(1) align 1 %dst, ptr addrspace(3) align 1 readonly %src, i64 %sz) {1043; CHECK-LABEL: memmove_p1_p3:1044; CHECK: ; %bb.0: ; %entry1045; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1046; CHECK-NEXT: v_and_b32_e32 v7, -16, v31047; CHECK-NEXT: v_mov_b32_e32 v8, v41048; CHECK-NEXT: v_and_b32_e32 v5, 15, v31049; CHECK-NEXT: v_mov_b32_e32 v6, 01050; CHECK-NEXT: s_mov_b64 s[4:5], 01051; CHECK-NEXT: s_mov_b32 s6, exec_lo1052; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]1053; CHECK-NEXT: s_cbranch_execz .LBB7_31054; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader1055; CHECK-NEXT: v_mov_b32_e32 v9, v21056; CHECK-NEXT: s_mov_b32 s7, 01057; CHECK-NEXT: .LBB7_2: ; %loop-memcpy-expansion1058; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11059; CHECK-NEXT: ds_read_b128 v[10:13], v91060; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s41061; CHECK-NEXT: s_add_u32 s4, s4, 161062; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo1063; CHECK-NEXT: s_addc_u32 s5, s5, 01064; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v91065; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]1066; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71067; CHECK-NEXT: s_waitcnt lgkmcnt(0)1068; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off1069; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71070; CHECK-NEXT: s_cbranch_execnz .LBB7_21071; CHECK-NEXT: .LBB7_3: ; %Flow91072; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61073; CHECK-NEXT: s_mov_b64 s[4:5], 01074; CHECK-NEXT: s_mov_b32 s6, exec_lo1075; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]1076; CHECK-NEXT: s_xor_b32 s6, exec_lo, s61077; CHECK-NEXT: s_cbranch_execz .LBB7_71078; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader1079; CHECK-NEXT: v_and_b32_e32 v3, -16, v31080; CHECK-NEXT: s_mov_b32 s7, 01081; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v31082; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v31083; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo1084; CHECK-NEXT: .LBB7_5: ; %loop-memcpy-residual1085; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11086; CHECK-NEXT: ds_read_u8 v7, v21087; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s41088; CHECK-NEXT: s_add_u32 s4, s4, 11089; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo1090; CHECK-NEXT: s_addc_u32 s5, s5, 01091; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v21092; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]1093; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71094; CHECK-NEXT: s_waitcnt lgkmcnt(0)1095; CHECK-NEXT: global_store_byte v[3:4], v7, off1096; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71097; CHECK-NEXT: s_cbranch_execnz .LBB7_51098; CHECK-NEXT: ; %bb.6: ; %Flow1099; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71100; CHECK-NEXT: .LBB7_7: ; %Flow71101; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61102; CHECK-NEXT: s_setpc_b64 s[30:31]1103entry:1104 tail call void @llvm.memmove.p1.p3.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 %sz, i1 false)1105 ret void1106}1107 1108define void @memmove_p1_p4(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align 1 readonly %src, i64 %sz) {1109; CHECK-LABEL: memmove_p1_p4:1110; CHECK: ; %bb.0: ; %entry1111; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1112; CHECK-NEXT: v_and_b32_e32 v6, 15, v41113; CHECK-NEXT: v_mov_b32_e32 v7, 01114; CHECK-NEXT: v_and_b32_e32 v8, -16, v41115; CHECK-NEXT: v_mov_b32_e32 v9, v51116; CHECK-NEXT: s_mov_b32 s6, exec_lo1117; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[6:7]1118; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9]1119; CHECK-NEXT: v_cmpx_ge_u64_e64 v[2:3], v[0:1]1120; CHECK-NEXT: s_xor_b32 s7, exec_lo, s61121; CHECK-NEXT: s_cbranch_execnz .LBB8_31122; CHECK-NEXT: ; %bb.1: ; %Flow381123; CHECK-NEXT: s_andn2_saveexec_b32 s6, s71124; CHECK-NEXT: s_cbranch_execnz .LBB8_101125; CHECK-NEXT: .LBB8_2: ; %Flow391126; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61127; CHECK-NEXT: s_setpc_b64 s[30:31]1128; CHECK-NEXT: .LBB8_3: ; %memmove_copy_forward1129; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo1130; CHECK-NEXT: s_cbranch_execz .LBB8_61131; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader1132; CHECK-NEXT: v_mov_b32_e32 v5, v31133; CHECK-NEXT: v_mov_b32_e32 v11, v11134; CHECK-NEXT: v_mov_b32_e32 v13, v91135; CHECK-NEXT: v_mov_b32_e32 v4, v21136; CHECK-NEXT: v_mov_b32_e32 v10, v01137; CHECK-NEXT: v_mov_b32_e32 v12, v81138; CHECK-NEXT: s_mov_b32 s9, 01139; CHECK-NEXT: .p2align 61140; CHECK-NEXT: .LBB8_5: ; %memmove_fwd_main_loop1141; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11142; CHECK-NEXT: global_load_dwordx4 v[14:17], v[4:5], off1143; CHECK-NEXT: v_add_co_u32 v12, s5, v12, -161144; CHECK-NEXT: v_add_co_ci_u32_e64 v13, null, -1, v13, s51145; CHECK-NEXT: v_add_co_u32 v4, s5, v4, 161146; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v5, s51147; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[12:13]1148; CHECK-NEXT: s_or_b32 s9, s5, s91149; CHECK-NEXT: s_waitcnt vmcnt(0)1150; CHECK-NEXT: global_store_dwordx4 v[10:11], v[14:17], off1151; CHECK-NEXT: v_add_co_u32 v10, s6, v10, 161152; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, s61153; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s91154; CHECK-NEXT: s_cbranch_execnz .LBB8_51155; CHECK-NEXT: .LBB8_6: ; %Flow331156; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s81157; CHECK-NEXT: s_and_saveexec_b32 s8, s41158; CHECK-NEXT: s_cbranch_execz .LBB8_91159; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader1160; CHECK-NEXT: v_add_co_u32 v0, s5, v0, v81161; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v9, s51162; CHECK-NEXT: v_add_co_u32 v2, s5, v2, v81163; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, v3, v9, s51164; CHECK-NEXT: s_mov_b32 s9, 01165; CHECK-NEXT: .p2align 61166; CHECK-NEXT: .LBB8_8: ; %memmove_fwd_residual_loop1167; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11168; CHECK-NEXT: global_load_ubyte v4, v[2:3], off1169; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -11170; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s51171; CHECK-NEXT: v_add_co_u32 v2, s5, v2, 11172; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s51173; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]1174; CHECK-NEXT: s_or_b32 s9, s5, s91175; CHECK-NEXT: s_waitcnt vmcnt(0)1176; CHECK-NEXT: global_store_byte v[0:1], v4, off1177; CHECK-NEXT: v_add_co_u32 v0, s6, v0, 11178; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s61179; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s91180; CHECK-NEXT: s_cbranch_execnz .LBB8_81181; CHECK-NEXT: .LBB8_9: ; %Flow311182; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s81183; CHECK-NEXT: ; implicit-def: $vgpr8_vgpr91184; CHECK-NEXT: ; implicit-def: $vgpr0_vgpr11185; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr31186; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr71187; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr51188; CHECK-NEXT: s_andn2_saveexec_b32 s6, s71189; CHECK-NEXT: s_cbranch_execz .LBB8_21190; CHECK-NEXT: .LBB8_10: ; %memmove_copy_backwards1191; CHECK-NEXT: s_and_saveexec_b32 s7, s41192; CHECK-NEXT: s_cbranch_execz .LBB8_131193; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader1194; CHECK-NEXT: v_add_co_u32 v10, s4, v4, -11195; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v5, s41196; CHECK-NEXT: s_mov_b32 s8, 01197; CHECK-NEXT: v_add_co_u32 v4, s4, v0, v101198; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v11, s41199; CHECK-NEXT: v_add_co_u32 v10, s4, v2, v101200; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v3, v11, s41201; CHECK-NEXT: .p2align 61202; CHECK-NEXT: .LBB8_12: ; %memmove_bwd_residual_loop1203; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11204; CHECK-NEXT: global_load_ubyte v12, v[10:11], off1205; CHECK-NEXT: v_add_co_u32 v6, s4, v6, -11206; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s41207; CHECK-NEXT: v_add_co_u32 v10, s4, v10, -11208; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, -1, v11, s41209; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[6:7]1210; CHECK-NEXT: s_or_b32 s8, s4, s81211; CHECK-NEXT: s_waitcnt vmcnt(0)1212; CHECK-NEXT: global_store_byte v[4:5], v12, off1213; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -11214; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s51215; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s81216; CHECK-NEXT: s_cbranch_execnz .LBB8_121217; CHECK-NEXT: .LBB8_13: ; %Flow371218; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71219; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo1220; CHECK-NEXT: s_cbranch_execz .LBB8_161221; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader1222; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, -161223; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo1224; CHECK-NEXT: v_add_co_u32 v2, vcc_lo, v2, -161225; CHECK-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo1226; CHECK-NEXT: s_mov_b32 s7, 01227; CHECK-NEXT: .p2align 61228; CHECK-NEXT: .LBB8_15: ; %memmove_bwd_main_loop1229; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11230; CHECK-NEXT: v_mov_b32_e32 v11, v91231; CHECK-NEXT: v_mov_b32_e32 v10, v81232; CHECK-NEXT: v_add_co_u32 v4, vcc_lo, v2, v101233; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, v3, v11, vcc_lo1234; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v10, -161235; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, -1, v11, vcc_lo1236; CHECK-NEXT: global_load_dwordx4 v[4:7], v[4:5], off1237; CHECK-NEXT: v_add_co_u32 v10, s4, v0, v101238; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[8:9]1239; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, v1, v11, s41240; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71241; CHECK-NEXT: s_waitcnt vmcnt(0)1242; CHECK-NEXT: global_store_dwordx4 v[10:11], v[4:7], off1243; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71244; CHECK-NEXT: s_cbranch_execnz .LBB8_151245; CHECK-NEXT: .LBB8_16: ; %Flow351246; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s51247; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61248; CHECK-NEXT: s_setpc_b64 s[30:31]1249entry:1250 tail call void @llvm.memmove.p1.p4.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 %sz, i1 false)1251 ret void1252}1253 1254define void @memmove_p1_p5(ptr addrspace(1) align 1 %dst, ptr addrspace(5) align 1 readonly %src, i64 %sz) {1255; CHECK-LABEL: memmove_p1_p5:1256; CHECK: ; %bb.0: ; %entry1257; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1258; CHECK-NEXT: v_and_b32_e32 v7, -16, v31259; CHECK-NEXT: v_mov_b32_e32 v8, v41260; CHECK-NEXT: v_and_b32_e32 v5, 15, v31261; CHECK-NEXT: v_mov_b32_e32 v6, 01262; CHECK-NEXT: s_mov_b64 s[4:5], 01263; CHECK-NEXT: s_mov_b32 s6, exec_lo1264; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]1265; CHECK-NEXT: s_cbranch_execz .LBB9_31266; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader1267; CHECK-NEXT: v_mov_b32_e32 v9, v21268; CHECK-NEXT: s_mov_b32 s7, 01269; CHECK-NEXT: .p2align 61270; CHECK-NEXT: .LBB9_2: ; %loop-memcpy-expansion1271; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11272; CHECK-NEXT: s_clause 0x31273; CHECK-NEXT: buffer_load_dword v10, v9, s[0:3], 0 offen1274; CHECK-NEXT: buffer_load_dword v11, v9, s[0:3], 0 offen offset:41275; CHECK-NEXT: buffer_load_dword v12, v9, s[0:3], 0 offen offset:81276; CHECK-NEXT: buffer_load_dword v13, v9, s[0:3], 0 offen offset:121277; CHECK-NEXT: v_add_co_u32 v14, vcc_lo, v0, s41278; CHECK-NEXT: s_add_u32 s4, s4, 161279; CHECK-NEXT: v_add_co_ci_u32_e64 v15, null, s5, v1, vcc_lo1280; CHECK-NEXT: s_addc_u32 s5, s5, 01281; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v91282; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]1283; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71284; CHECK-NEXT: s_waitcnt vmcnt(0)1285; CHECK-NEXT: global_store_dwordx4 v[14:15], v[10:13], off1286; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71287; CHECK-NEXT: s_cbranch_execnz .LBB9_21288; CHECK-NEXT: .LBB9_3: ; %Flow91289; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61290; CHECK-NEXT: s_mov_b64 s[4:5], 01291; CHECK-NEXT: s_mov_b32 s6, exec_lo1292; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]1293; CHECK-NEXT: s_xor_b32 s6, exec_lo, s61294; CHECK-NEXT: s_cbranch_execz .LBB9_71295; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader1296; CHECK-NEXT: v_and_b32_e32 v3, -16, v31297; CHECK-NEXT: s_mov_b32 s7, 01298; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, v31299; CHECK-NEXT: v_add_nc_u32_e32 v2, v2, v31300; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo1301; CHECK-NEXT: .LBB9_5: ; %loop-memcpy-residual1302; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11303; CHECK-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen1304; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v0, s41305; CHECK-NEXT: s_add_u32 s4, s4, 11306; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v1, vcc_lo1307; CHECK-NEXT: s_addc_u32 s5, s5, 01308; CHECK-NEXT: v_add_nc_u32_e32 v2, 1, v21309; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]1310; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71311; CHECK-NEXT: s_waitcnt vmcnt(0)1312; CHECK-NEXT: global_store_byte v[3:4], v7, off1313; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71314; CHECK-NEXT: s_cbranch_execnz .LBB9_51315; CHECK-NEXT: ; %bb.6: ; %Flow1316; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71317; CHECK-NEXT: .LBB9_7: ; %Flow71318; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61319; CHECK-NEXT: s_setpc_b64 s[30:31]1320entry:1321 tail call void @llvm.memmove.p1.p5.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 %sz, i1 false)1322 ret void1323}1324 1325 1326define void @memmove_p3_p0(ptr addrspace(3) align 1 %dst, ptr addrspace(0) align 1 readonly %src, i64 %sz) {1327; CHECK-LABEL: memmove_p3_p0:1328; CHECK: ; %bb.0: ; %entry1329; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1330; CHECK-NEXT: v_and_b32_e32 v5, 15, v31331; CHECK-NEXT: v_mov_b32_e32 v6, 01332; CHECK-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v01333; CHECK-NEXT: s_mov_b64 s[4:5], src_shared_base1334; CHECK-NEXT: v_and_b32_e32 v7, -16, v31335; CHECK-NEXT: v_mov_b32_e32 v8, v41336; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]1337; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, s5, vcc_lo1338; CHECK-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc_lo1339; CHECK-NEXT: s_mov_b32 s6, exec_lo1340; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[7:8]1341; CHECK-NEXT: v_cmpx_ge_u64_e64 v[1:2], v[9:10]1342; CHECK-NEXT: s_xor_b32 s7, exec_lo, s61343; CHECK-NEXT: s_cbranch_execnz .LBB10_31344; CHECK-NEXT: ; %bb.1: ; %Flow391345; CHECK-NEXT: s_andn2_saveexec_b32 s6, s71346; CHECK-NEXT: s_cbranch_execnz .LBB10_101347; CHECK-NEXT: .LBB10_2: ; %Flow401348; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61349; CHECK-NEXT: s_waitcnt lgkmcnt(0)1350; CHECK-NEXT: s_setpc_b64 s[30:31]1351; CHECK-NEXT: .LBB10_3: ; %memmove_copy_forward1352; CHECK-NEXT: s_and_saveexec_b32 s8, vcc_lo1353; CHECK-NEXT: s_cbranch_execz .LBB10_61354; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader1355; CHECK-NEXT: v_mov_b32_e32 v10, v21356; CHECK-NEXT: v_mov_b32_e32 v12, v81357; CHECK-NEXT: v_mov_b32_e32 v9, v11358; CHECK-NEXT: v_mov_b32_e32 v11, v71359; CHECK-NEXT: v_mov_b32_e32 v4, v01360; CHECK-NEXT: s_mov_b32 s9, 01361; CHECK-NEXT: .p2align 61362; CHECK-NEXT: .LBB10_5: ; %memmove_fwd_main_loop1363; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11364; CHECK-NEXT: flat_load_dwordx4 v[13:16], v[9:10]1365; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -161366; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s51367; CHECK-NEXT: v_add_co_u32 v9, s5, v9, 161368; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s51369; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[11:12]1370; CHECK-NEXT: s_or_b32 s9, s6, s91371; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1372; CHECK-NEXT: ds_write_b128 v4, v[13:16]1373; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v41374; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s91375; CHECK-NEXT: s_cbranch_execnz .LBB10_51376; CHECK-NEXT: .LBB10_6: ; %Flow341377; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s81378; CHECK-NEXT: s_and_saveexec_b32 s8, s41379; CHECK-NEXT: s_cbranch_execz .LBB10_91380; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader1381; CHECK-NEXT: v_and_b32_e32 v3, -16, v31382; CHECK-NEXT: s_mov_b32 s9, 01383; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v31384; CHECK-NEXT: v_add_co_u32 v0, s5, v1, v71385; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v8, s51386; CHECK-NEXT: .p2align 61387; CHECK-NEXT: .LBB10_8: ; %memmove_fwd_residual_loop1388; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11389; CHECK-NEXT: flat_load_ubyte v2, v[0:1]1390; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -11391; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s51392; CHECK-NEXT: v_add_co_u32 v0, s5, v0, 11393; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s51394; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[5:6]1395; CHECK-NEXT: s_or_b32 s9, s6, s91396; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1397; CHECK-NEXT: ds_write_b8 v3, v21398; CHECK-NEXT: v_add_nc_u32_e32 v3, 1, v31399; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s91400; CHECK-NEXT: s_cbranch_execnz .LBB10_81401; CHECK-NEXT: .LBB10_9: ; %Flow321402; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s81403; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr81404; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr41405; CHECK-NEXT: ; implicit-def: $vgpr01406; CHECK-NEXT: ; implicit-def: $vgpr1_vgpr21407; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr61408; CHECK-NEXT: s_andn2_saveexec_b32 s6, s71409; CHECK-NEXT: s_cbranch_execz .LBB10_21410; CHECK-NEXT: .LBB10_10: ; %memmove_copy_backwards1411; CHECK-NEXT: s_and_saveexec_b32 s7, s41412; CHECK-NEXT: s_cbranch_execz .LBB10_131413; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader1414; CHECK-NEXT: v_add_co_u32 v9, s4, v1, v31415; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v2, v4, s41416; CHECK-NEXT: v_add3_u32 v4, v3, v0, -11417; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -11418; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s41419; CHECK-NEXT: s_mov_b32 s8, 01420; CHECK-NEXT: .p2align 61421; CHECK-NEXT: .LBB10_12: ; %memmove_bwd_residual_loop1422; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11423; CHECK-NEXT: flat_load_ubyte v11, v[9:10]1424; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -11425; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s41426; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -11427; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s41428; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]1429; CHECK-NEXT: s_or_b32 s8, s5, s81430; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1431; CHECK-NEXT: ds_write_b8 v4, v111432; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v41433; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s81434; CHECK-NEXT: s_cbranch_execnz .LBB10_121435; CHECK-NEXT: .LBB10_13: ; %Flow381436; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71437; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo1438; CHECK-NEXT: s_cbranch_execz .LBB10_161439; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader1440; CHECK-NEXT: v_and_b32_e32 v3, -16, v31441; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, -161442; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, -1, v2, vcc_lo1443; CHECK-NEXT: v_add3_u32 v0, v3, v0, -161444; CHECK-NEXT: s_mov_b32 s5, 01445; CHECK-NEXT: .p2align 61446; CHECK-NEXT: .LBB10_15: ; %memmove_bwd_main_loop1447; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11448; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, v71449; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v2, v8, vcc_lo1450; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v7, -161451; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, vcc_lo1452; CHECK-NEXT: flat_load_dwordx4 v[3:6], v[3:4]1453; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[7:8]1454; CHECK-NEXT: s_or_b32 s5, vcc_lo, s51455; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1456; CHECK-NEXT: ds_write_b128 v0, v[3:6]1457; CHECK-NEXT: v_add_nc_u32_e32 v0, -16, v01458; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s51459; CHECK-NEXT: s_cbranch_execnz .LBB10_151460; CHECK-NEXT: .LBB10_16: ; %Flow361461; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s41462; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61463; CHECK-NEXT: s_waitcnt lgkmcnt(0)1464; CHECK-NEXT: s_setpc_b64 s[30:31]1465entry:1466 tail call void @llvm.memmove.p3.p0.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 %sz, i1 false)1467 ret void1468}1469 1470define void @memmove_p3_p1(ptr addrspace(3) align 1 %dst, ptr addrspace(1) align 1 readonly %src, i64 %sz) {1471; CHECK-LABEL: memmove_p3_p1:1472; CHECK: ; %bb.0: ; %entry1473; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1474; CHECK-NEXT: v_and_b32_e32 v7, -16, v31475; CHECK-NEXT: v_mov_b32_e32 v8, v41476; CHECK-NEXT: v_and_b32_e32 v5, 15, v31477; CHECK-NEXT: v_mov_b32_e32 v6, 01478; CHECK-NEXT: s_mov_b64 s[4:5], 01479; CHECK-NEXT: s_mov_b32 s6, exec_lo1480; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]1481; CHECK-NEXT: s_cbranch_execz .LBB11_31482; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader1483; CHECK-NEXT: v_mov_b32_e32 v9, v01484; CHECK-NEXT: s_mov_b32 s7, 01485; CHECK-NEXT: .LBB11_2: ; %loop-memcpy-expansion1486; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11487; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s41488; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo1489; CHECK-NEXT: s_add_u32 s4, s4, 161490; CHECK-NEXT: s_addc_u32 s5, s5, 01491; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]1492; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off1493; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71494; CHECK-NEXT: s_waitcnt vmcnt(0)1495; CHECK-NEXT: ds_write_b128 v9, v[10:13]1496; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v91497; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71498; CHECK-NEXT: s_cbranch_execnz .LBB11_21499; CHECK-NEXT: .LBB11_3: ; %Flow91500; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61501; CHECK-NEXT: s_mov_b64 s[4:5], 01502; CHECK-NEXT: s_mov_b32 s6, exec_lo1503; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]1504; CHECK-NEXT: s_xor_b32 s6, exec_lo, s61505; CHECK-NEXT: s_cbranch_execz .LBB11_71506; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader1507; CHECK-NEXT: v_and_b32_e32 v3, -16, v31508; CHECK-NEXT: s_mov_b32 s7, 01509; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v31510; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v31511; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo1512; CHECK-NEXT: .LBB11_5: ; %loop-memcpy-residual1513; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11514; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s41515; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo1516; CHECK-NEXT: s_add_u32 s4, s4, 11517; CHECK-NEXT: s_addc_u32 s5, s5, 01518; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]1519; CHECK-NEXT: global_load_ubyte v3, v[3:4], off1520; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71521; CHECK-NEXT: s_waitcnt vmcnt(0)1522; CHECK-NEXT: ds_write_b8 v0, v31523; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v01524; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71525; CHECK-NEXT: s_cbranch_execnz .LBB11_51526; CHECK-NEXT: ; %bb.6: ; %Flow1527; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71528; CHECK-NEXT: .LBB11_7: ; %Flow71529; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61530; CHECK-NEXT: s_waitcnt lgkmcnt(0)1531; CHECK-NEXT: s_setpc_b64 s[30:31]1532entry:1533 tail call void @llvm.memmove.p3.p1.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 %sz, i1 false)1534 ret void1535}1536 1537define void @memmove_p3_p3(ptr addrspace(3) align 1 %dst, ptr addrspace(3) align 1 readonly %src, i64 %sz) {1538; CHECK-LABEL: memmove_p3_p3:1539; CHECK: ; %bb.0: ; %entry1540; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1541; CHECK-NEXT: v_mov_b32_e32 v5, 01542; CHECK-NEXT: v_and_b32_e32 v4, 15, v21543; CHECK-NEXT: v_and_b32_e32 v6, -16, v21544; CHECK-NEXT: v_mov_b32_e32 v7, v31545; CHECK-NEXT: s_mov_b32 s6, exec_lo1546; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]1547; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]1548; CHECK-NEXT: v_cmpx_ge_u32_e64 v1, v01549; CHECK-NEXT: s_xor_b32 s6, exec_lo, s61550; CHECK-NEXT: s_cbranch_execnz .LBB12_31551; CHECK-NEXT: ; %bb.1: ; %Flow461552; CHECK-NEXT: s_andn2_saveexec_b32 s5, s61553; CHECK-NEXT: s_cbranch_execnz .LBB12_101554; CHECK-NEXT: .LBB12_2: ; %Flow471555; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s51556; CHECK-NEXT: s_waitcnt lgkmcnt(0)1557; CHECK-NEXT: s_setpc_b64 s[30:31]1558; CHECK-NEXT: .LBB12_3: ; %memmove_copy_forward1559; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo1560; CHECK-NEXT: s_cbranch_execz .LBB12_61561; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader1562; CHECK-NEXT: v_mov_b32_e32 v3, v11563; CHECK-NEXT: v_mov_b32_e32 v8, v01564; CHECK-NEXT: s_mov_b32 s8, 01565; CHECK-NEXT: .LBB12_5: ; %memmove_fwd_main_loop1566; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11567; CHECK-NEXT: ds_read_b128 v[9:12], v31568; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -161569; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s51570; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v31571; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]1572; CHECK-NEXT: s_or_b32 s8, s5, s81573; CHECK-NEXT: s_waitcnt lgkmcnt(0)1574; CHECK-NEXT: ds_write_b128 v8, v[9:12]1575; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v81576; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s81577; CHECK-NEXT: s_cbranch_execnz .LBB12_51578; CHECK-NEXT: .LBB12_6: ; %Flow411579; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71580; CHECK-NEXT: s_and_saveexec_b32 s7, s41581; CHECK-NEXT: s_cbranch_execz .LBB12_91582; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader1583; CHECK-NEXT: v_and_b32_e32 v2, -16, v21584; CHECK-NEXT: s_mov_b32 s8, 01585; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v21586; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v21587; CHECK-NEXT: .LBB12_8: ; %memmove_fwd_residual_loop1588; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11589; CHECK-NEXT: ds_read_u8 v2, v11590; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -11591; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s51592; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v11593; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]1594; CHECK-NEXT: s_or_b32 s8, s5, s81595; CHECK-NEXT: s_waitcnt lgkmcnt(0)1596; CHECK-NEXT: ds_write_b8 v0, v21597; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v01598; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s81599; CHECK-NEXT: s_cbranch_execnz .LBB12_81600; CHECK-NEXT: .LBB12_9: ; %Flow391601; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71602; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr31603; CHECK-NEXT: ; implicit-def: $vgpr01604; CHECK-NEXT: ; implicit-def: $vgpr11605; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr51606; CHECK-NEXT: s_andn2_saveexec_b32 s5, s61607; CHECK-NEXT: s_cbranch_execz .LBB12_21608; CHECK-NEXT: .LBB12_10: ; %memmove_copy_backwards1609; CHECK-NEXT: s_and_saveexec_b32 s6, s41610; CHECK-NEXT: s_cbranch_execz .LBB12_131611; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader1612; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v21613; CHECK-NEXT: s_mov_b32 s7, 01614; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v71615; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v71616; CHECK-NEXT: .LBB12_12: ; %memmove_bwd_residual_loop1617; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11618; CHECK-NEXT: ds_read_u8 v8, v71619; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -11620; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s41621; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v71622; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]1623; CHECK-NEXT: s_or_b32 s7, s4, s71624; CHECK-NEXT: s_waitcnt lgkmcnt(0)1625; CHECK-NEXT: ds_write_b8 v6, v81626; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v61627; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71628; CHECK-NEXT: s_cbranch_execnz .LBB12_121629; CHECK-NEXT: .LBB12_13: ; %Flow451630; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61631; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo1632; CHECK-NEXT: s_cbranch_execz .LBB12_161633; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader1634; CHECK-NEXT: v_and_b32_e32 v5, -16, v21635; CHECK-NEXT: s_mov_b32 s6, 01636; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v51637; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v41638; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v51639; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v41640; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo1641; CHECK-NEXT: .LBB12_15: ; %memmove_bwd_main_loop1642; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11643; CHECK-NEXT: ds_read_b128 v[5:8], v41644; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 161645; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo1646; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v41647; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]1648; CHECK-NEXT: s_or_b32 s6, vcc_lo, s61649; CHECK-NEXT: s_waitcnt lgkmcnt(0)1650; CHECK-NEXT: ds_write_b128 v2, v[5:8]1651; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v21652; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s61653; CHECK-NEXT: s_cbranch_execnz .LBB12_151654; CHECK-NEXT: .LBB12_16: ; %Flow431655; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s41656; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s51657; CHECK-NEXT: s_waitcnt lgkmcnt(0)1658; CHECK-NEXT: s_setpc_b64 s[30:31]1659entry:1660 tail call void @llvm.memmove.p3.p3.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 %sz, i1 false)1661 ret void1662}1663 1664define void @memmove_p3_p4(ptr addrspace(3) align 1 %dst, ptr addrspace(4) align 1 readonly %src, i64 %sz) {1665; CHECK-LABEL: memmove_p3_p4:1666; CHECK: ; %bb.0: ; %entry1667; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1668; CHECK-NEXT: v_and_b32_e32 v7, -16, v31669; CHECK-NEXT: v_mov_b32_e32 v8, v41670; CHECK-NEXT: v_and_b32_e32 v5, 15, v31671; CHECK-NEXT: v_mov_b32_e32 v6, 01672; CHECK-NEXT: s_mov_b64 s[4:5], 01673; CHECK-NEXT: s_mov_b32 s6, exec_lo1674; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]1675; CHECK-NEXT: s_cbranch_execz .LBB13_31676; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader1677; CHECK-NEXT: v_mov_b32_e32 v9, v01678; CHECK-NEXT: s_mov_b32 s7, 01679; CHECK-NEXT: .LBB13_2: ; %loop-memcpy-expansion1680; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11681; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s41682; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo1683; CHECK-NEXT: s_add_u32 s4, s4, 161684; CHECK-NEXT: s_addc_u32 s5, s5, 01685; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]1686; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off1687; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71688; CHECK-NEXT: s_waitcnt vmcnt(0)1689; CHECK-NEXT: ds_write_b128 v9, v[10:13]1690; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v91691; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71692; CHECK-NEXT: s_cbranch_execnz .LBB13_21693; CHECK-NEXT: .LBB13_3: ; %Flow91694; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61695; CHECK-NEXT: s_mov_b64 s[4:5], 01696; CHECK-NEXT: s_mov_b32 s6, exec_lo1697; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]1698; CHECK-NEXT: s_xor_b32 s6, exec_lo, s61699; CHECK-NEXT: s_cbranch_execz .LBB13_71700; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader1701; CHECK-NEXT: v_and_b32_e32 v3, -16, v31702; CHECK-NEXT: s_mov_b32 s7, 01703; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v31704; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v31705; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo1706; CHECK-NEXT: .LBB13_5: ; %loop-memcpy-residual1707; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11708; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s41709; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo1710; CHECK-NEXT: s_add_u32 s4, s4, 11711; CHECK-NEXT: s_addc_u32 s5, s5, 01712; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]1713; CHECK-NEXT: global_load_ubyte v3, v[3:4], off1714; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71715; CHECK-NEXT: s_waitcnt vmcnt(0)1716; CHECK-NEXT: ds_write_b8 v0, v31717; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v01718; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71719; CHECK-NEXT: s_cbranch_execnz .LBB13_51720; CHECK-NEXT: ; %bb.6: ; %Flow1721; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71722; CHECK-NEXT: .LBB13_7: ; %Flow71723; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61724; CHECK-NEXT: s_waitcnt lgkmcnt(0)1725; CHECK-NEXT: s_setpc_b64 s[30:31]1726entry:1727 tail call void @llvm.memmove.p3.p4.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 %sz, i1 false)1728 ret void1729}1730 1731define void @memmove_p3_p5(ptr addrspace(3) align 1 %dst, ptr addrspace(5) align 1 readonly %src, i64 %sz) {1732; CHECK-LABEL: memmove_p3_p5:1733; CHECK: ; %bb.0: ; %entry1734; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1735; CHECK-NEXT: v_mov_b32_e32 v4, v21736; CHECK-NEXT: v_mov_b32_e32 v6, 01737; CHECK-NEXT: s_mov_b64 s[4:5], 01738; CHECK-NEXT: s_mov_b32 s6, exec_lo1739; CHECK-NEXT: v_and_b32_e32 v2, -16, v41740; CHECK-NEXT: v_and_b32_e32 v5, 15, v41741; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[2:3]1742; CHECK-NEXT: s_cbranch_execz .LBB14_31743; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader1744; CHECK-NEXT: v_mov_b32_e32 v7, v11745; CHECK-NEXT: v_mov_b32_e32 v8, v01746; CHECK-NEXT: s_mov_b32 s7, 01747; CHECK-NEXT: .p2align 61748; CHECK-NEXT: .LBB14_2: ; %loop-memcpy-expansion1749; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11750; CHECK-NEXT: s_clause 0x31751; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen1752; CHECK-NEXT: buffer_load_dword v10, v7, s[0:3], 0 offen offset:41753; CHECK-NEXT: buffer_load_dword v11, v7, s[0:3], 0 offen offset:81754; CHECK-NEXT: buffer_load_dword v12, v7, s[0:3], 0 offen offset:121755; CHECK-NEXT: s_add_u32 s4, s4, 161756; CHECK-NEXT: s_addc_u32 s5, s5, 01757; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v71758; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[2:3]1759; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71760; CHECK-NEXT: s_waitcnt vmcnt(0)1761; CHECK-NEXT: ds_write_b128 v8, v[9:12]1762; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v81763; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71764; CHECK-NEXT: s_cbranch_execnz .LBB14_21765; CHECK-NEXT: .LBB14_3: ; %Flow141766; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61767; CHECK-NEXT: s_mov_b64 s[4:5], 01768; CHECK-NEXT: s_mov_b32 s6, exec_lo1769; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]1770; CHECK-NEXT: s_xor_b32 s6, exec_lo, s61771; CHECK-NEXT: s_cbranch_execz .LBB14_71772; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader1773; CHECK-NEXT: v_and_b32_e32 v2, -16, v41774; CHECK-NEXT: s_mov_b32 s7, 01775; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v21776; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v21777; CHECK-NEXT: .LBB14_5: ; %loop-memcpy-residual1778; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11779; CHECK-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen1780; CHECK-NEXT: s_add_u32 s4, s4, 11781; CHECK-NEXT: s_addc_u32 s5, s5, 01782; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v11783; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]1784; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71785; CHECK-NEXT: s_waitcnt vmcnt(0)1786; CHECK-NEXT: ds_write_b8 v0, v21787; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v01788; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71789; CHECK-NEXT: s_cbranch_execnz .LBB14_51790; CHECK-NEXT: ; %bb.6: ; %Flow1791; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71792; CHECK-NEXT: .LBB14_7: ; %Flow121793; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61794; CHECK-NEXT: s_waitcnt lgkmcnt(0)1795; CHECK-NEXT: s_setpc_b64 s[30:31]1796entry:1797 tail call void @llvm.memmove.p3.p5.i64(ptr addrspace(3) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 %sz, i1 false)1798 ret void1799}1800 1801 1802define void @memmove_p5_p0(ptr addrspace(5) align 1 %dst, ptr addrspace(0) align 1 readonly %src, i64 %sz) {1803; CHECK-LABEL: memmove_p5_p0:1804; CHECK: ; %bb.0: ; %entry1805; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1806; CHECK-NEXT: v_and_b32_e32 v5, 15, v31807; CHECK-NEXT: v_mov_b32_e32 v6, 01808; CHECK-NEXT: v_cmp_ne_u32_e32 vcc_lo, -1, v01809; CHECK-NEXT: s_mov_b64 s[4:5], src_private_base1810; CHECK-NEXT: v_and_b32_e32 v7, -16, v31811; CHECK-NEXT: v_mov_b32_e32 v8, v41812; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[5:6]1813; CHECK-NEXT: v_cndmask_b32_e64 v10, 0, s5, vcc_lo1814; CHECK-NEXT: v_cndmask_b32_e32 v9, 0, v0, vcc_lo1815; CHECK-NEXT: s_mov_b32 s6, exec_lo1816; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[7:8]1817; CHECK-NEXT: v_cmpx_ge_u64_e64 v[1:2], v[9:10]1818; CHECK-NEXT: s_xor_b32 s7, exec_lo, s61819; CHECK-NEXT: s_cbranch_execnz .LBB15_31820; CHECK-NEXT: ; %bb.1: ; %Flow391821; CHECK-NEXT: s_andn2_saveexec_b32 s6, s71822; CHECK-NEXT: s_cbranch_execnz .LBB15_101823; CHECK-NEXT: .LBB15_2: ; %Flow401824; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61825; CHECK-NEXT: s_setpc_b64 s[30:31]1826; CHECK-NEXT: .LBB15_3: ; %memmove_copy_forward1827; CHECK-NEXT: s_and_saveexec_b32 s6, vcc_lo1828; CHECK-NEXT: s_cbranch_execz .LBB15_61829; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader1830; CHECK-NEXT: v_mov_b32_e32 v10, v21831; CHECK-NEXT: v_mov_b32_e32 v12, v81832; CHECK-NEXT: v_mov_b32_e32 v9, v11833; CHECK-NEXT: v_mov_b32_e32 v11, v71834; CHECK-NEXT: v_mov_b32_e32 v4, v01835; CHECK-NEXT: s_mov_b32 s8, 01836; CHECK-NEXT: .p2align 61837; CHECK-NEXT: .LBB15_5: ; %memmove_fwd_main_loop1838; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11839; CHECK-NEXT: flat_load_dwordx4 v[13:16], v[9:10]1840; CHECK-NEXT: v_add_co_u32 v11, s5, v11, -161841; CHECK-NEXT: v_add_co_ci_u32_e64 v12, null, -1, v12, s51842; CHECK-NEXT: v_add_co_u32 v9, s5, v9, 161843; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, 0, v10, s51844; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[11:12]1845; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1846; CHECK-NEXT: buffer_store_dword v16, v4, s[0:3], 0 offen offset:121847; CHECK-NEXT: buffer_store_dword v15, v4, s[0:3], 0 offen offset:81848; CHECK-NEXT: buffer_store_dword v14, v4, s[0:3], 0 offen offset:41849; CHECK-NEXT: buffer_store_dword v13, v4, s[0:3], 0 offen1850; CHECK-NEXT: v_add_nc_u32_e32 v4, 16, v41851; CHECK-NEXT: s_or_b32 s8, s5, s81852; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s81853; CHECK-NEXT: s_cbranch_execnz .LBB15_51854; CHECK-NEXT: .LBB15_6: ; %Flow341855; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61856; CHECK-NEXT: s_and_saveexec_b32 s8, s41857; CHECK-NEXT: s_cbranch_execz .LBB15_91858; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader1859; CHECK-NEXT: v_and_b32_e32 v3, -16, v31860; CHECK-NEXT: s_mov_b32 s9, 01861; CHECK-NEXT: v_add_nc_u32_e32 v3, v0, v31862; CHECK-NEXT: v_add_co_u32 v0, s5, v1, v71863; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v8, s51864; CHECK-NEXT: .p2align 61865; CHECK-NEXT: .LBB15_8: ; %memmove_fwd_residual_loop1866; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11867; CHECK-NEXT: flat_load_ubyte v2, v[0:1]1868; CHECK-NEXT: v_add_co_u32 v5, s5, v5, -11869; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s51870; CHECK-NEXT: v_add_co_u32 v0, s5, v0, 11871; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s51872; CHECK-NEXT: v_cmp_eq_u64_e64 s6, 0, v[5:6]1873; CHECK-NEXT: s_or_b32 s9, s6, s91874; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1875; CHECK-NEXT: buffer_store_byte v2, v3, s[0:3], 0 offen1876; CHECK-NEXT: v_add_nc_u32_e32 v3, 1, v31877; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s91878; CHECK-NEXT: s_cbranch_execnz .LBB15_81879; CHECK-NEXT: .LBB15_9: ; %Flow321880; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s81881; CHECK-NEXT: ; implicit-def: $vgpr7_vgpr81882; CHECK-NEXT: ; implicit-def: $vgpr3_vgpr41883; CHECK-NEXT: ; implicit-def: $vgpr01884; CHECK-NEXT: ; implicit-def: $vgpr1_vgpr21885; CHECK-NEXT: ; implicit-def: $vgpr5_vgpr61886; CHECK-NEXT: s_andn2_saveexec_b32 s6, s71887; CHECK-NEXT: s_cbranch_execz .LBB15_21888; CHECK-NEXT: .LBB15_10: ; %memmove_copy_backwards1889; CHECK-NEXT: s_and_saveexec_b32 s7, s41890; CHECK-NEXT: s_cbranch_execz .LBB15_131891; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader1892; CHECK-NEXT: v_add_co_u32 v9, s4, v1, v31893; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, v2, v4, s41894; CHECK-NEXT: v_add3_u32 v4, v3, v0, -11895; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -11896; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s41897; CHECK-NEXT: s_mov_b32 s8, 01898; CHECK-NEXT: .p2align 61899; CHECK-NEXT: .LBB15_12: ; %memmove_bwd_residual_loop1900; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11901; CHECK-NEXT: flat_load_ubyte v11, v[9:10]1902; CHECK-NEXT: v_add_co_u32 v5, s4, v5, -11903; CHECK-NEXT: v_add_co_ci_u32_e64 v6, null, -1, v6, s41904; CHECK-NEXT: v_add_co_u32 v9, s4, v9, -11905; CHECK-NEXT: v_add_co_ci_u32_e64 v10, null, -1, v10, s41906; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[5:6]1907; CHECK-NEXT: s_or_b32 s8, s5, s81908; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1909; CHECK-NEXT: buffer_store_byte v11, v4, s[0:3], 0 offen1910; CHECK-NEXT: v_add_nc_u32_e32 v4, -1, v41911; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s81912; CHECK-NEXT: s_cbranch_execnz .LBB15_121913; CHECK-NEXT: .LBB15_13: ; %Flow381914; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s71915; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo1916; CHECK-NEXT: s_cbranch_execz .LBB15_161917; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader1918; CHECK-NEXT: v_and_b32_e32 v3, -16, v31919; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, -161920; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, -1, v2, vcc_lo1921; CHECK-NEXT: v_add3_u32 v0, v3, v0, -161922; CHECK-NEXT: s_mov_b32 s5, 01923; CHECK-NEXT: .p2align 61924; CHECK-NEXT: .LBB15_15: ; %memmove_bwd_main_loop1925; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11926; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, v71927; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, v2, v8, vcc_lo1928; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v7, -161929; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, -1, v8, vcc_lo1930; CHECK-NEXT: flat_load_dwordx4 v[3:6], v[3:4]1931; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1932; CHECK-NEXT: buffer_store_dword v6, v0, s[0:3], 0 offen offset:121933; CHECK-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen offset:81934; CHECK-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen offset:41935; CHECK-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen1936; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[7:8]1937; CHECK-NEXT: v_add_nc_u32_e32 v0, -16, v01938; CHECK-NEXT: s_or_b32 s5, vcc_lo, s51939; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s51940; CHECK-NEXT: s_cbranch_execnz .LBB15_151941; CHECK-NEXT: .LBB15_16: ; %Flow361942; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s41943; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61944; CHECK-NEXT: s_setpc_b64 s[30:31]1945entry:1946 tail call void @llvm.memmove.p5.p0.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(0) noundef nonnull align 1 %src, i64 %sz, i1 false)1947 ret void1948}1949 1950define void @memmove_p5_p1(ptr addrspace(5) align 1 %dst, ptr addrspace(1) align 1 readonly %src, i64 %sz) {1951; CHECK-LABEL: memmove_p5_p1:1952; CHECK: ; %bb.0: ; %entry1953; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1954; CHECK-NEXT: v_and_b32_e32 v7, -16, v31955; CHECK-NEXT: v_mov_b32_e32 v8, v41956; CHECK-NEXT: v_and_b32_e32 v5, 15, v31957; CHECK-NEXT: v_mov_b32_e32 v6, 01958; CHECK-NEXT: s_mov_b64 s[4:5], 01959; CHECK-NEXT: s_mov_b32 s6, exec_lo1960; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]1961; CHECK-NEXT: s_cbranch_execz .LBB16_31962; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader1963; CHECK-NEXT: v_mov_b32_e32 v9, v01964; CHECK-NEXT: s_mov_b32 s7, 01965; CHECK-NEXT: .p2align 61966; CHECK-NEXT: .LBB16_2: ; %loop-memcpy-expansion1967; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11968; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s41969; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo1970; CHECK-NEXT: s_add_u32 s4, s4, 161971; CHECK-NEXT: s_addc_u32 s5, s5, 01972; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]1973; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off1974; CHECK-NEXT: s_waitcnt vmcnt(0)1975; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:121976; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:81977; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:41978; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen1979; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v91980; CHECK-NEXT: s_or_b32 s7, vcc_lo, s71981; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s71982; CHECK-NEXT: s_cbranch_execnz .LBB16_21983; CHECK-NEXT: .LBB16_3: ; %Flow91984; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s61985; CHECK-NEXT: s_mov_b64 s[4:5], 01986; CHECK-NEXT: s_mov_b32 s6, exec_lo1987; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]1988; CHECK-NEXT: s_xor_b32 s6, exec_lo, s61989; CHECK-NEXT: s_cbranch_execz .LBB16_71990; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader1991; CHECK-NEXT: v_and_b32_e32 v3, -16, v31992; CHECK-NEXT: s_mov_b32 s7, 01993; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v31994; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v31995; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo1996; CHECK-NEXT: .LBB16_5: ; %loop-memcpy-residual1997; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11998; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s41999; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo2000; CHECK-NEXT: s_add_u32 s4, s4, 12001; CHECK-NEXT: s_addc_u32 s5, s5, 02002; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]2003; CHECK-NEXT: global_load_ubyte v3, v[3:4], off2004; CHECK-NEXT: s_or_b32 s7, vcc_lo, s72005; CHECK-NEXT: s_waitcnt vmcnt(0)2006; CHECK-NEXT: buffer_store_byte v3, v0, s[0:3], 0 offen2007; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v02008; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s72009; CHECK-NEXT: s_cbranch_execnz .LBB16_52010; CHECK-NEXT: ; %bb.6: ; %Flow2011; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s72012; CHECK-NEXT: .LBB16_7: ; %Flow72013; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s62014; CHECK-NEXT: s_setpc_b64 s[30:31]2015entry:2016 tail call void @llvm.memmove.p5.p1.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(1) noundef nonnull align 1 %src, i64 %sz, i1 false)2017 ret void2018}2019 2020define void @memmove_p5_p3(ptr addrspace(5) align 1 %dst, ptr addrspace(3) align 1 readonly %src, i64 %sz) {2021; CHECK-LABEL: memmove_p5_p3:2022; CHECK: ; %bb.0: ; %entry2023; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2024; CHECK-NEXT: v_mov_b32_e32 v4, v22025; CHECK-NEXT: v_mov_b32_e32 v6, 02026; CHECK-NEXT: s_mov_b64 s[4:5], 02027; CHECK-NEXT: s_mov_b32 s6, exec_lo2028; CHECK-NEXT: v_and_b32_e32 v2, -16, v42029; CHECK-NEXT: v_and_b32_e32 v5, 15, v42030; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[2:3]2031; CHECK-NEXT: s_cbranch_execz .LBB17_32032; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader2033; CHECK-NEXT: v_mov_b32_e32 v7, v12034; CHECK-NEXT: v_mov_b32_e32 v8, v02035; CHECK-NEXT: s_mov_b32 s7, 02036; CHECK-NEXT: .p2align 62037; CHECK-NEXT: .LBB17_2: ; %loop-memcpy-expansion2038; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12039; CHECK-NEXT: ds_read_b128 v[9:12], v72040; CHECK-NEXT: s_add_u32 s4, s4, 162041; CHECK-NEXT: s_addc_u32 s5, s5, 02042; CHECK-NEXT: v_add_nc_u32_e32 v7, 16, v72043; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[2:3]2044; CHECK-NEXT: s_waitcnt lgkmcnt(0)2045; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen offset:122046; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:82047; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:42048; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen2049; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v82050; CHECK-NEXT: s_or_b32 s7, vcc_lo, s72051; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s72052; CHECK-NEXT: s_cbranch_execnz .LBB17_22053; CHECK-NEXT: .LBB17_3: ; %Flow142054; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s62055; CHECK-NEXT: s_mov_b64 s[4:5], 02056; CHECK-NEXT: s_mov_b32 s6, exec_lo2057; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]2058; CHECK-NEXT: s_xor_b32 s6, exec_lo, s62059; CHECK-NEXT: s_cbranch_execz .LBB17_72060; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader2061; CHECK-NEXT: v_and_b32_e32 v2, -16, v42062; CHECK-NEXT: s_mov_b32 s7, 02063; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v22064; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v22065; CHECK-NEXT: .LBB17_5: ; %loop-memcpy-residual2066; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12067; CHECK-NEXT: ds_read_u8 v2, v12068; CHECK-NEXT: s_add_u32 s4, s4, 12069; CHECK-NEXT: s_addc_u32 s5, s5, 02070; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v12071; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]2072; CHECK-NEXT: s_or_b32 s7, vcc_lo, s72073; CHECK-NEXT: s_waitcnt lgkmcnt(0)2074; CHECK-NEXT: buffer_store_byte v2, v0, s[0:3], 0 offen2075; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v02076; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s72077; CHECK-NEXT: s_cbranch_execnz .LBB17_52078; CHECK-NEXT: ; %bb.6: ; %Flow2079; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s72080; CHECK-NEXT: .LBB17_7: ; %Flow122081; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s62082; CHECK-NEXT: s_setpc_b64 s[30:31]2083entry:2084 tail call void @llvm.memmove.p5.p3.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(3) noundef nonnull align 1 %src, i64 %sz, i1 false)2085 ret void2086}2087 2088define void @memmove_p5_p4(ptr addrspace(5) align 1 %dst, ptr addrspace(4) align 1 readonly %src, i64 %sz) {2089; CHECK-LABEL: memmove_p5_p4:2090; CHECK: ; %bb.0: ; %entry2091; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2092; CHECK-NEXT: v_and_b32_e32 v7, -16, v32093; CHECK-NEXT: v_mov_b32_e32 v8, v42094; CHECK-NEXT: v_and_b32_e32 v5, 15, v32095; CHECK-NEXT: v_mov_b32_e32 v6, 02096; CHECK-NEXT: s_mov_b64 s[4:5], 02097; CHECK-NEXT: s_mov_b32 s6, exec_lo2098; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[7:8]2099; CHECK-NEXT: s_cbranch_execz .LBB18_32100; CHECK-NEXT: ; %bb.1: ; %loop-memcpy-expansion.preheader2101; CHECK-NEXT: v_mov_b32_e32 v9, v02102; CHECK-NEXT: s_mov_b32 s7, 02103; CHECK-NEXT: .p2align 62104; CHECK-NEXT: .LBB18_2: ; %loop-memcpy-expansion2105; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12106; CHECK-NEXT: v_add_co_u32 v10, vcc_lo, v1, s42107; CHECK-NEXT: v_add_co_ci_u32_e64 v11, null, s5, v2, vcc_lo2108; CHECK-NEXT: s_add_u32 s4, s4, 162109; CHECK-NEXT: s_addc_u32 s5, s5, 02110; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[7:8]2111; CHECK-NEXT: global_load_dwordx4 v[10:13], v[10:11], off2112; CHECK-NEXT: s_waitcnt vmcnt(0)2113; CHECK-NEXT: buffer_store_dword v13, v9, s[0:3], 0 offen offset:122114; CHECK-NEXT: buffer_store_dword v12, v9, s[0:3], 0 offen offset:82115; CHECK-NEXT: buffer_store_dword v11, v9, s[0:3], 0 offen offset:42116; CHECK-NEXT: buffer_store_dword v10, v9, s[0:3], 0 offen2117; CHECK-NEXT: v_add_nc_u32_e32 v9, 16, v92118; CHECK-NEXT: s_or_b32 s7, vcc_lo, s72119; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s72120; CHECK-NEXT: s_cbranch_execnz .LBB18_22121; CHECK-NEXT: .LBB18_3: ; %Flow92122; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s62123; CHECK-NEXT: s_mov_b64 s[4:5], 02124; CHECK-NEXT: s_mov_b32 s6, exec_lo2125; CHECK-NEXT: v_cmpx_ne_u64_e32 0, v[5:6]2126; CHECK-NEXT: s_xor_b32 s6, exec_lo, s62127; CHECK-NEXT: s_cbranch_execz .LBB18_72128; CHECK-NEXT: ; %bb.4: ; %loop-memcpy-residual.preheader2129; CHECK-NEXT: v_and_b32_e32 v3, -16, v32130; CHECK-NEXT: s_mov_b32 s7, 02131; CHECK-NEXT: v_add_co_u32 v1, vcc_lo, v1, v32132; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v32133; CHECK-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo2134; CHECK-NEXT: .LBB18_5: ; %loop-memcpy-residual2135; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12136; CHECK-NEXT: v_add_co_u32 v3, vcc_lo, v1, s42137; CHECK-NEXT: v_add_co_ci_u32_e64 v4, null, s5, v2, vcc_lo2138; CHECK-NEXT: s_add_u32 s4, s4, 12139; CHECK-NEXT: s_addc_u32 s5, s5, 02140; CHECK-NEXT: v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[5:6]2141; CHECK-NEXT: global_load_ubyte v3, v[3:4], off2142; CHECK-NEXT: s_or_b32 s7, vcc_lo, s72143; CHECK-NEXT: s_waitcnt vmcnt(0)2144; CHECK-NEXT: buffer_store_byte v3, v0, s[0:3], 0 offen2145; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v02146; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s72147; CHECK-NEXT: s_cbranch_execnz .LBB18_52148; CHECK-NEXT: ; %bb.6: ; %Flow2149; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s72150; CHECK-NEXT: .LBB18_7: ; %Flow72151; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s62152; CHECK-NEXT: s_setpc_b64 s[30:31]2153entry:2154 tail call void @llvm.memmove.p5.p4.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 %sz, i1 false)2155 ret void2156}2157 2158define void @memmove_p5_p5(ptr addrspace(5) align 1 %dst, ptr addrspace(5) align 1 readonly %src, i64 %sz) {2159; CHECK-LABEL: memmove_p5_p5:2160; CHECK: ; %bb.0: ; %entry2161; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2162; CHECK-NEXT: v_mov_b32_e32 v5, 02163; CHECK-NEXT: v_and_b32_e32 v4, 15, v22164; CHECK-NEXT: v_and_b32_e32 v6, -16, v22165; CHECK-NEXT: v_mov_b32_e32 v7, v32166; CHECK-NEXT: s_mov_b32 s6, exec_lo2167; CHECK-NEXT: v_cmp_ne_u64_e64 s4, 0, v[4:5]2168; CHECK-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]2169; CHECK-NEXT: v_cmpx_ge_u32_e64 v1, v02170; CHECK-NEXT: s_xor_b32 s6, exec_lo, s62171; CHECK-NEXT: s_cbranch_execnz .LBB19_32172; CHECK-NEXT: ; %bb.1: ; %Flow462173; CHECK-NEXT: s_andn2_saveexec_b32 s5, s62174; CHECK-NEXT: s_cbranch_execnz .LBB19_102175; CHECK-NEXT: .LBB19_2: ; %Flow472176; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s52177; CHECK-NEXT: s_setpc_b64 s[30:31]2178; CHECK-NEXT: .LBB19_3: ; %memmove_copy_forward2179; CHECK-NEXT: s_and_saveexec_b32 s7, vcc_lo2180; CHECK-NEXT: s_cbranch_execz .LBB19_62181; CHECK-NEXT: ; %bb.4: ; %memmove_fwd_main_loop.preheader2182; CHECK-NEXT: v_mov_b32_e32 v3, v12183; CHECK-NEXT: v_mov_b32_e32 v8, v02184; CHECK-NEXT: s_mov_b32 s8, 02185; CHECK-NEXT: .p2align 62186; CHECK-NEXT: .LBB19_5: ; %memmove_fwd_main_loop2187; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12188; CHECK-NEXT: s_clause 0x32189; CHECK-NEXT: buffer_load_dword v9, v3, s[0:3], 0 offen offset:122190; CHECK-NEXT: buffer_load_dword v10, v3, s[0:3], 0 offen offset:82191; CHECK-NEXT: buffer_load_dword v11, v3, s[0:3], 0 offen offset:42192; CHECK-NEXT: buffer_load_dword v12, v3, s[0:3], 0 offen2193; CHECK-NEXT: v_add_co_u32 v6, s5, v6, -162194; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v7, s52195; CHECK-NEXT: v_add_nc_u32_e32 v3, 16, v32196; CHECK-NEXT: s_waitcnt vmcnt(3)2197; CHECK-NEXT: buffer_store_dword v9, v8, s[0:3], 0 offen offset:122198; CHECK-NEXT: s_waitcnt vmcnt(2)2199; CHECK-NEXT: buffer_store_dword v10, v8, s[0:3], 0 offen offset:82200; CHECK-NEXT: s_waitcnt vmcnt(1)2201; CHECK-NEXT: buffer_store_dword v11, v8, s[0:3], 0 offen offset:42202; CHECK-NEXT: s_waitcnt vmcnt(0)2203; CHECK-NEXT: buffer_store_dword v12, v8, s[0:3], 0 offen2204; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[6:7]2205; CHECK-NEXT: v_add_nc_u32_e32 v8, 16, v82206; CHECK-NEXT: s_or_b32 s8, s5, s82207; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s82208; CHECK-NEXT: s_cbranch_execnz .LBB19_52209; CHECK-NEXT: .LBB19_6: ; %Flow412210; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s72211; CHECK-NEXT: s_and_saveexec_b32 s7, s42212; CHECK-NEXT: s_cbranch_execz .LBB19_92213; CHECK-NEXT: ; %bb.7: ; %memmove_fwd_residual_loop.preheader2214; CHECK-NEXT: v_and_b32_e32 v2, -16, v22215; CHECK-NEXT: s_mov_b32 s8, 02216; CHECK-NEXT: v_add_nc_u32_e32 v0, v0, v22217; CHECK-NEXT: v_add_nc_u32_e32 v1, v1, v22218; CHECK-NEXT: .LBB19_8: ; %memmove_fwd_residual_loop2219; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12220; CHECK-NEXT: buffer_load_ubyte v2, v1, s[0:3], 0 offen2221; CHECK-NEXT: v_add_co_u32 v4, s5, v4, -12222; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s52223; CHECK-NEXT: v_add_nc_u32_e32 v1, 1, v12224; CHECK-NEXT: v_cmp_eq_u64_e64 s5, 0, v[4:5]2225; CHECK-NEXT: s_or_b32 s8, s5, s82226; CHECK-NEXT: s_waitcnt vmcnt(0)2227; CHECK-NEXT: buffer_store_byte v2, v0, s[0:3], 0 offen2228; CHECK-NEXT: v_add_nc_u32_e32 v0, 1, v02229; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s82230; CHECK-NEXT: s_cbranch_execnz .LBB19_82231; CHECK-NEXT: .LBB19_9: ; %Flow392232; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s72233; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr32234; CHECK-NEXT: ; implicit-def: $vgpr02235; CHECK-NEXT: ; implicit-def: $vgpr12236; CHECK-NEXT: ; implicit-def: $vgpr4_vgpr52237; CHECK-NEXT: s_andn2_saveexec_b32 s5, s62238; CHECK-NEXT: s_cbranch_execz .LBB19_22239; CHECK-NEXT: .LBB19_10: ; %memmove_copy_backwards2240; CHECK-NEXT: s_and_saveexec_b32 s6, s42241; CHECK-NEXT: s_cbranch_execz .LBB19_132242; CHECK-NEXT: ; %bb.11: ; %memmove_bwd_residual_loop.preheader2243; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v22244; CHECK-NEXT: s_mov_b32 s7, 02245; CHECK-NEXT: v_add_nc_u32_e32 v6, v0, v72246; CHECK-NEXT: v_add_nc_u32_e32 v7, v1, v72247; CHECK-NEXT: .LBB19_12: ; %memmove_bwd_residual_loop2248; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12249; CHECK-NEXT: buffer_load_ubyte v8, v7, s[0:3], 0 offen2250; CHECK-NEXT: v_add_co_u32 v4, s4, v4, -12251; CHECK-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v5, s42252; CHECK-NEXT: v_add_nc_u32_e32 v7, -1, v72253; CHECK-NEXT: v_cmp_eq_u64_e64 s4, 0, v[4:5]2254; CHECK-NEXT: s_or_b32 s7, s4, s72255; CHECK-NEXT: s_waitcnt vmcnt(0)2256; CHECK-NEXT: buffer_store_byte v8, v6, s[0:3], 0 offen2257; CHECK-NEXT: v_add_nc_u32_e32 v6, -1, v62258; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s72259; CHECK-NEXT: s_cbranch_execnz .LBB19_122260; CHECK-NEXT: .LBB19_13: ; %Flow452261; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s62262; CHECK-NEXT: s_and_saveexec_b32 s4, vcc_lo2263; CHECK-NEXT: s_cbranch_execz .LBB19_162264; CHECK-NEXT: ; %bb.14: ; %memmove_bwd_main_loop.preheader2265; CHECK-NEXT: v_and_b32_e32 v5, -16, v22266; CHECK-NEXT: s_mov_b32 s6, 02267; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v52268; CHECK-NEXT: v_add_nc_u32_e32 v2, v0, v42269; CHECK-NEXT: v_sub_co_u32 v0, vcc_lo, 0, v52270; CHECK-NEXT: v_add_nc_u32_e32 v4, v1, v42271; CHECK-NEXT: v_sub_co_ci_u32_e64 v1, null, 0, v3, vcc_lo2272; CHECK-NEXT: .p2align 62273; CHECK-NEXT: .LBB19_15: ; %memmove_bwd_main_loop2274; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12275; CHECK-NEXT: s_clause 0x32276; CHECK-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:122277; CHECK-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen offset:82278; CHECK-NEXT: buffer_load_dword v6, v4, s[0:3], 0 offen offset:42279; CHECK-NEXT: buffer_load_dword v7, v4, s[0:3], 0 offen2280; CHECK-NEXT: v_add_co_u32 v0, vcc_lo, v0, 162281; CHECK-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo2282; CHECK-NEXT: v_add_nc_u32_e32 v4, -16, v42283; CHECK-NEXT: s_waitcnt vmcnt(3)2284; CHECK-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:122285; CHECK-NEXT: s_waitcnt vmcnt(2)2286; CHECK-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:82287; CHECK-NEXT: s_waitcnt vmcnt(1)2288; CHECK-NEXT: buffer_store_dword v6, v2, s[0:3], 0 offen offset:42289; CHECK-NEXT: s_waitcnt vmcnt(0)2290; CHECK-NEXT: buffer_store_dword v7, v2, s[0:3], 0 offen2291; CHECK-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]2292; CHECK-NEXT: v_add_nc_u32_e32 v2, -16, v22293; CHECK-NEXT: s_or_b32 s6, vcc_lo, s62294; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s62295; CHECK-NEXT: s_cbranch_execnz .LBB19_152296; CHECK-NEXT: .LBB19_16: ; %Flow432297; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s42298; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s52299; CHECK-NEXT: s_setpc_b64 s[30:31]2300entry:2301 tail call void @llvm.memmove.p5.p5.i64(ptr addrspace(5) noundef nonnull align 1 %dst, ptr addrspace(5) noundef nonnull align 1 %src, i64 %sz, i1 false)2302 ret void2303}2304 2305 2306declare void @llvm.memmove.p0.p0.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(0) nocapture readonly, i64, i1 immarg) #02307declare void @llvm.memmove.p0.p1.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(1) nocapture readonly, i64, i1 immarg) #02308declare void @llvm.memmove.p0.p3.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(3) nocapture readonly, i64, i1 immarg) #02309declare void @llvm.memmove.p0.p4.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(4) nocapture readonly, i64, i1 immarg) #02310declare void @llvm.memmove.p0.p5.i64(ptr addrspace(0) nocapture writeonly, ptr addrspace(5) nocapture readonly, i64, i1 immarg) #02311declare void @llvm.memmove.p1.p0.i64(ptr addrspace(1) nocapture writeonly, ptr addrspace(0) nocapture readonly, i64, i1 immarg) #02312declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) nocapture writeonly, ptr addrspace(1) nocapture readonly, i64, i1 immarg) #02313declare void @llvm.memmove.p1.p3.i64(ptr addrspace(1) nocapture writeonly, ptr addrspace(3) nocapture readonly, i64, i1 immarg) #02314declare void @llvm.memmove.p1.p4.i64(ptr addrspace(1) nocapture writeonly, ptr addrspace(4) nocapture readonly, i64, i1 immarg) #02315declare void @llvm.memmove.p1.p5.i64(ptr addrspace(1) nocapture writeonly, ptr addrspace(5) nocapture readonly, i64, i1 immarg) #02316declare void @llvm.memmove.p3.p0.i64(ptr addrspace(3) nocapture writeonly, ptr addrspace(0) nocapture readonly, i64, i1 immarg) #02317declare void @llvm.memmove.p3.p1.i64(ptr addrspace(3) nocapture writeonly, ptr addrspace(1) nocapture readonly, i64, i1 immarg) #02318declare void @llvm.memmove.p3.p3.i64(ptr addrspace(3) nocapture writeonly, ptr addrspace(3) nocapture readonly, i64, i1 immarg) #02319declare void @llvm.memmove.p3.p4.i64(ptr addrspace(3) nocapture writeonly, ptr addrspace(4) nocapture readonly, i64, i1 immarg) #02320declare void @llvm.memmove.p3.p5.i64(ptr addrspace(3) nocapture writeonly, ptr addrspace(5) nocapture readonly, i64, i1 immarg) #02321declare void @llvm.memmove.p5.p0.i64(ptr addrspace(5) nocapture writeonly, ptr addrspace(0) nocapture readonly, i64, i1 immarg) #02322declare void @llvm.memmove.p5.p1.i64(ptr addrspace(5) nocapture writeonly, ptr addrspace(1) nocapture readonly, i64, i1 immarg) #02323declare void @llvm.memmove.p5.p3.i64(ptr addrspace(5) nocapture writeonly, ptr addrspace(3) nocapture readonly, i64, i1 immarg) #02324declare void @llvm.memmove.p5.p4.i64(ptr addrspace(5) nocapture writeonly, ptr addrspace(4) nocapture readonly, i64, i1 immarg) #02325declare void @llvm.memmove.p5.p5.i64(ptr addrspace(5) nocapture writeonly, ptr addrspace(5) nocapture readonly, i64, i1 immarg) #02326 2327attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }2328