720 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti | FileCheck %s --check-prefix=SI3; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefix=VI4; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 | FileCheck %s --check-prefix=GFX95; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1010 | FileCheck %s --check-prefix=GFX106; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s --check-prefix=GFX117 8 9declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone10declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) nounwind readnone11 12 13declare { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32>, <2 x i32>) nounwind readnone14 15define amdgpu_kernel void @saddo_i64_zext(ptr addrspace(1) %out, i64 %a, i64 %b) nounwind {16; SI-LABEL: saddo_i64_zext:17; SI: ; %bb.0:18; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x919; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd20; SI-NEXT: s_mov_b32 s7, 0xf00021; SI-NEXT: s_mov_b32 s6, -122; SI-NEXT: s_waitcnt lgkmcnt(0)23; SI-NEXT: v_mov_b32_e32 v0, s224; SI-NEXT: s_add_u32 s10, s2, s825; SI-NEXT: s_addc_u32 s11, s3, s926; SI-NEXT: v_mov_b32_e32 v1, s327; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]28; SI-NEXT: v_cmp_lt_i64_e64 s[2:3], s[8:9], 029; SI-NEXT: s_mov_b32 s4, s030; SI-NEXT: s_mov_b32 s5, s131; SI-NEXT: s_xor_b64 s[0:1], s[2:3], vcc32; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]33; SI-NEXT: v_mov_b32_e32 v1, s1134; SI-NEXT: v_add_i32_e32 v0, vcc, s10, v035; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc36; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 037; SI-NEXT: s_endpgm38;39; VI-LABEL: saddo_i64_zext:40; VI: ; %bb.0:41; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2442; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3443; VI-NEXT: s_waitcnt lgkmcnt(0)44; VI-NEXT: v_mov_b32_e32 v1, s245; VI-NEXT: s_add_u32 s6, s2, s446; VI-NEXT: v_mov_b32_e32 v2, s347; VI-NEXT: s_addc_u32 s7, s3, s548; VI-NEXT: v_cmp_lt_i64_e64 s[8:9], s[4:5], 049; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[1:2]50; VI-NEXT: v_mov_b32_e32 v0, s051; VI-NEXT: v_mov_b32_e32 v1, s152; VI-NEXT: s_xor_b64 s[0:1], s[8:9], vcc53; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]54; VI-NEXT: v_mov_b32_e32 v3, s755; VI-NEXT: v_add_u32_e32 v2, vcc, s6, v256; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc57; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]58; VI-NEXT: s_endpgm59;60; GFX9-LABEL: saddo_i64_zext:61; GFX9: ; %bb.0:62; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2463; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3464; GFX9-NEXT: v_mov_b32_e32 v2, 065; GFX9-NEXT: s_waitcnt lgkmcnt(0)66; GFX9-NEXT: v_mov_b32_e32 v0, s267; GFX9-NEXT: s_add_u32 s4, s2, s668; GFX9-NEXT: v_mov_b32_e32 v1, s369; GFX9-NEXT: s_addc_u32 s5, s3, s770; GFX9-NEXT: v_cmp_lt_i64_e64 s[8:9], s[6:7], 071; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]72; GFX9-NEXT: v_mov_b32_e32 v1, s573; GFX9-NEXT: s_xor_b64 s[2:3], s[8:9], vcc74; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]75; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v076; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc77; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]78; GFX9-NEXT: s_endpgm79;80; GFX10-LABEL: saddo_i64_zext:81; GFX10: ; %bb.0:82; GFX10-NEXT: s_clause 0x183; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2484; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3485; GFX10-NEXT: v_mov_b32_e32 v2, 086; GFX10-NEXT: s_waitcnt lgkmcnt(0)87; GFX10-NEXT: s_add_u32 s4, s2, s688; GFX10-NEXT: s_addc_u32 s5, s3, s789; GFX10-NEXT: v_cmp_lt_i64_e64 s6, s[6:7], 090; GFX10-NEXT: v_cmp_lt_i64_e64 s2, s[4:5], s[2:3]91; GFX10-NEXT: s_xor_b32 s2, s6, s292; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s293; GFX10-NEXT: v_add_co_u32 v0, s2, s4, v094; GFX10-NEXT: v_add_co_ci_u32_e64 v1, s2, s5, 0, s295; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]96; GFX10-NEXT: s_endpgm97;98; GFX11-LABEL: saddo_i64_zext:99; GFX11: ; %bb.0:100; GFX11-NEXT: s_clause 0x1101; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24102; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34103; GFX11-NEXT: v_mov_b32_e32 v2, 0104; GFX11-NEXT: s_waitcnt lgkmcnt(0)105; GFX11-NEXT: s_add_u32 s6, s2, s4106; GFX11-NEXT: s_addc_u32 s7, s3, s5107; GFX11-NEXT: v_cmp_lt_i64_e64 s4, s[4:5], 0108; GFX11-NEXT: v_cmp_lt_i64_e64 s2, s[6:7], s[2:3]109; GFX11-NEXT: s_xor_b32 s2, s4, s2110; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)111; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2112; GFX11-NEXT: v_add_co_u32 v0, s2, s6, v0113; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)114; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s2115; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]116; GFX11-NEXT: s_endpgm117 %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind118 %val = extractvalue { i64, i1 } %sadd, 0119 %carry = extractvalue { i64, i1 } %sadd, 1120 %ext = zext i1 %carry to i64121 %add2 = add i64 %val, %ext122 store i64 %add2, ptr addrspace(1) %out, align 8123 ret void124}125 126define amdgpu_kernel void @s_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %carryout, i32 %a, i32 %b) nounwind {127; SI-LABEL: s_saddo_i32:128; SI: ; %bb.0:129; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9130; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd131; SI-NEXT: s_mov_b32 s7, 0xf000132; SI-NEXT: s_mov_b32 s6, -1133; SI-NEXT: s_waitcnt lgkmcnt(0)134; SI-NEXT: s_mov_b32 s4, s0135; SI-NEXT: s_add_i32 s12, s8, s9136; SI-NEXT: s_cmp_lt_i32 s9, 0137; SI-NEXT: s_cselect_b64 s[10:11], -1, 0138; SI-NEXT: s_cmp_lt_i32 s12, s8139; SI-NEXT: s_mov_b32 s5, s1140; SI-NEXT: s_cselect_b64 s[8:9], -1, 0141; SI-NEXT: v_mov_b32_e32 v0, s12142; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0143; SI-NEXT: s_xor_b64 s[4:5], s[10:11], s[8:9]144; SI-NEXT: s_mov_b32 s0, s2145; SI-NEXT: s_mov_b32 s1, s3146; SI-NEXT: s_mov_b32 s2, s6147; SI-NEXT: s_mov_b32 s3, s7148; SI-NEXT: s_waitcnt expcnt(0)149; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]150; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0151; SI-NEXT: s_endpgm152;153; VI-LABEL: s_saddo_i32:154; VI: ; %bb.0:155; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24156; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34157; VI-NEXT: s_waitcnt lgkmcnt(0)158; VI-NEXT: v_mov_b32_e32 v0, s0159; VI-NEXT: s_add_i32 s6, s4, s5160; VI-NEXT: s_cmp_lt_i32 s5, 0161; VI-NEXT: v_mov_b32_e32 v1, s1162; VI-NEXT: s_cselect_b64 s[0:1], -1, 0163; VI-NEXT: s_cmp_lt_i32 s6, s4164; VI-NEXT: v_mov_b32_e32 v2, s2165; VI-NEXT: v_mov_b32_e32 v3, s3166; VI-NEXT: s_cselect_b64 s[2:3], -1, 0167; VI-NEXT: v_mov_b32_e32 v4, s6168; VI-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]169; VI-NEXT: flat_store_dword v[0:1], v4170; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]171; VI-NEXT: flat_store_byte v[2:3], v0172; VI-NEXT: s_endpgm173;174; GFX9-LABEL: s_saddo_i32:175; GFX9: ; %bb.0:176; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34177; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24178; GFX9-NEXT: v_mov_b32_e32 v0, 0179; GFX9-NEXT: s_waitcnt lgkmcnt(0)180; GFX9-NEXT: v_mov_b32_e32 v1, s7181; GFX9-NEXT: s_add_i32 s4, s6, s7182; GFX9-NEXT: v_add_i32 v1, s6, v1 clamp183; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1184; GFX9-NEXT: v_mov_b32_e32 v2, s4185; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc186; GFX9-NEXT: global_store_dword v0, v2, s[0:1]187; GFX9-NEXT: global_store_byte v0, v1, s[2:3]188; GFX9-NEXT: s_endpgm189;190; GFX10-LABEL: s_saddo_i32:191; GFX10: ; %bb.0:192; GFX10-NEXT: s_clause 0x1193; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34194; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24195; GFX10-NEXT: v_mov_b32_e32 v1, 0196; GFX10-NEXT: s_waitcnt lgkmcnt(0)197; GFX10-NEXT: v_add_nc_i32 v0, s6, s7 clamp198; GFX10-NEXT: s_add_i32 s4, s6, s7199; GFX10-NEXT: v_mov_b32_e32 v2, s4200; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v0201; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo202; GFX10-NEXT: global_store_dword v1, v2, s[0:1]203; GFX10-NEXT: global_store_byte v1, v0, s[2:3]204; GFX10-NEXT: s_endpgm205;206; GFX11-LABEL: s_saddo_i32:207; GFX11: ; %bb.0:208; GFX11-NEXT: s_clause 0x1209; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34210; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24211; GFX11-NEXT: s_waitcnt lgkmcnt(0)212; GFX11-NEXT: v_add_nc_i32 v0, s6, s7 clamp213; GFX11-NEXT: s_add_i32 s4, s6, s7214; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)215; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4216; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v0217; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo218; GFX11-NEXT: s_clause 0x1219; GFX11-NEXT: global_store_b32 v1, v2, s[0:1]220; GFX11-NEXT: global_store_b8 v1, v0, s[2:3]221; GFX11-NEXT: s_endpgm222 %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind223 %val = extractvalue { i32, i1 } %sadd, 0224 %carry = extractvalue { i32, i1 } %sadd, 1225 store i32 %val, ptr addrspace(1) %out, align 4226 store i1 %carry, ptr addrspace(1) %carryout227 ret void228}229 230define amdgpu_kernel void @v_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %carryout, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) nounwind {231; SI-LABEL: v_saddo_i32:232; SI: ; %bb.0:233; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9234; SI-NEXT: s_mov_b32 s11, 0xf000235; SI-NEXT: s_mov_b32 s10, -1236; SI-NEXT: s_mov_b32 s14, s10237; SI-NEXT: s_mov_b32 s15, s11238; SI-NEXT: s_waitcnt lgkmcnt(0)239; SI-NEXT: s_mov_b32 s12, s4240; SI-NEXT: s_mov_b32 s13, s5241; SI-NEXT: s_mov_b32 s4, s6242; SI-NEXT: s_mov_b32 s5, s7243; SI-NEXT: s_mov_b32 s6, s10244; SI-NEXT: s_mov_b32 s7, s11245; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0246; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0247; SI-NEXT: s_mov_b32 s8, s0248; SI-NEXT: s_mov_b32 s9, s1249; SI-NEXT: s_mov_b32 s4, s2250; SI-NEXT: s_mov_b32 s5, s3251; SI-NEXT: s_waitcnt vmcnt(0)252; SI-NEXT: v_add_i32_e32 v2, vcc, v0, v1253; SI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1254; SI-NEXT: v_cmp_lt_i32_e64 s[0:1], v2, v0255; SI-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]256; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]257; SI-NEXT: buffer_store_dword v2, off, s[8:11], 0258; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0259; SI-NEXT: s_endpgm260;261; VI-LABEL: v_saddo_i32:262; VI: ; %bb.0:263; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24264; VI-NEXT: s_waitcnt lgkmcnt(0)265; VI-NEXT: v_mov_b32_e32 v0, s4266; VI-NEXT: v_mov_b32_e32 v1, s5267; VI-NEXT: v_mov_b32_e32 v2, s6268; VI-NEXT: v_mov_b32_e32 v3, s7269; VI-NEXT: flat_load_dword v4, v[0:1]270; VI-NEXT: flat_load_dword v5, v[2:3]271; VI-NEXT: v_mov_b32_e32 v0, s0272; VI-NEXT: v_mov_b32_e32 v1, s1273; VI-NEXT: v_mov_b32_e32 v2, s2274; VI-NEXT: v_mov_b32_e32 v3, s3275; VI-NEXT: s_waitcnt vmcnt(0)276; VI-NEXT: v_add_u32_e32 v6, vcc, v4, v5277; VI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v5278; VI-NEXT: v_cmp_lt_i32_e64 s[0:1], v6, v4279; VI-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]280; VI-NEXT: flat_store_dword v[0:1], v6281; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]282; VI-NEXT: flat_store_byte v[2:3], v0283; VI-NEXT: s_endpgm284;285; GFX9-LABEL: v_saddo_i32:286; GFX9: ; %bb.0:287; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24288; GFX9-NEXT: v_mov_b32_e32 v0, 0289; GFX9-NEXT: s_waitcnt lgkmcnt(0)290; GFX9-NEXT: global_load_dword v1, v0, s[12:13]291; GFX9-NEXT: global_load_dword v2, v0, s[14:15]292; GFX9-NEXT: s_waitcnt vmcnt(0)293; GFX9-NEXT: v_add_i32 v3, v1, v2 clamp294; GFX9-NEXT: v_add_u32_e32 v1, v1, v2295; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v1, v3296; GFX9-NEXT: global_store_dword v0, v1, s[8:9]297; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc298; GFX9-NEXT: global_store_byte v0, v1, s[10:11]299; GFX9-NEXT: s_endpgm300;301; GFX10-LABEL: v_saddo_i32:302; GFX10: ; %bb.0:303; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24304; GFX10-NEXT: v_mov_b32_e32 v0, 0305; GFX10-NEXT: s_waitcnt lgkmcnt(0)306; GFX10-NEXT: s_clause 0x1307; GFX10-NEXT: global_load_dword v1, v0, s[12:13]308; GFX10-NEXT: global_load_dword v2, v0, s[14:15]309; GFX10-NEXT: s_waitcnt vmcnt(0)310; GFX10-NEXT: v_add_nc_i32 v3, v1, v2 clamp311; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v2312; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v3313; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo314; GFX10-NEXT: global_store_dword v0, v1, s[8:9]315; GFX10-NEXT: global_store_byte v0, v2, s[10:11]316; GFX10-NEXT: s_endpgm317;318; GFX11-LABEL: v_saddo_i32:319; GFX11: ; %bb.0:320; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24321; GFX11-NEXT: v_mov_b32_e32 v0, 0322; GFX11-NEXT: s_waitcnt lgkmcnt(0)323; GFX11-NEXT: s_clause 0x1324; GFX11-NEXT: global_load_b32 v1, v0, s[4:5]325; GFX11-NEXT: global_load_b32 v2, v0, s[6:7]326; GFX11-NEXT: s_waitcnt vmcnt(0)327; GFX11-NEXT: v_add_nc_i32 v3, v1, v2 clamp328; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v2329; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)330; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v3331; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo332; GFX11-NEXT: s_clause 0x1333; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]334; GFX11-NEXT: global_store_b8 v0, v2, s[2:3]335; GFX11-NEXT: s_endpgm336 %a = load i32, ptr addrspace(1) %aptr, align 4337 %b = load i32, ptr addrspace(1) %bptr, align 4338 %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind339 %val = extractvalue { i32, i1 } %sadd, 0340 %carry = extractvalue { i32, i1 } %sadd, 1341 store i32 %val, ptr addrspace(1) %out, align 4342 store i1 %carry, ptr addrspace(1) %carryout343 ret void344}345 346define amdgpu_kernel void @s_saddo_i64(ptr addrspace(1) %out, ptr addrspace(1) %carryout, i64 %a, i64 %b) nounwind {347; SI-LABEL: s_saddo_i64:348; SI: ; %bb.0:349; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9350; SI-NEXT: s_mov_b32 s11, 0xf000351; SI-NEXT: s_mov_b32 s10, -1352; SI-NEXT: s_waitcnt lgkmcnt(0)353; SI-NEXT: s_add_u32 s12, s4, s6354; SI-NEXT: v_mov_b32_e32 v0, s4355; SI-NEXT: s_addc_u32 s13, s5, s7356; SI-NEXT: v_mov_b32_e32 v1, s5357; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]358; SI-NEXT: v_cmp_lt_i64_e64 s[4:5], s[6:7], 0359; SI-NEXT: v_mov_b32_e32 v0, s12360; SI-NEXT: s_mov_b32 s8, s0361; SI-NEXT: s_mov_b32 s9, s1362; SI-NEXT: v_mov_b32_e32 v1, s13363; SI-NEXT: s_xor_b64 s[4:5], s[4:5], vcc364; SI-NEXT: s_mov_b32 s0, s2365; SI-NEXT: s_mov_b32 s1, s3366; SI-NEXT: s_mov_b32 s2, s10367; SI-NEXT: s_mov_b32 s3, s11368; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0369; SI-NEXT: s_waitcnt expcnt(0)370; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]371; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0372; SI-NEXT: s_endpgm373;374; VI-LABEL: s_saddo_i64:375; VI: ; %bb.0:376; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24377; VI-NEXT: s_waitcnt lgkmcnt(0)378; VI-NEXT: v_mov_b32_e32 v0, s0379; VI-NEXT: s_add_u32 s0, s4, s6380; VI-NEXT: v_mov_b32_e32 v4, s4381; VI-NEXT: v_mov_b32_e32 v1, s1382; VI-NEXT: s_addc_u32 s1, s5, s7383; VI-NEXT: v_mov_b32_e32 v5, s5384; VI-NEXT: v_mov_b32_e32 v2, s2385; VI-NEXT: v_mov_b32_e32 v3, s3386; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]387; VI-NEXT: v_cmp_lt_i64_e64 s[2:3], s[6:7], 0388; VI-NEXT: v_mov_b32_e32 v5, s1389; VI-NEXT: v_mov_b32_e32 v4, s0390; VI-NEXT: s_xor_b64 s[0:1], s[2:3], vcc391; VI-NEXT: flat_store_dwordx2 v[0:1], v[4:5]392; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]393; VI-NEXT: flat_store_byte v[2:3], v0394; VI-NEXT: s_endpgm395;396; GFX9-LABEL: s_saddo_i64:397; GFX9: ; %bb.0:398; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24399; GFX9-NEXT: v_mov_b32_e32 v2, 0400; GFX9-NEXT: s_waitcnt lgkmcnt(0)401; GFX9-NEXT: s_add_u32 s0, s12, s14402; GFX9-NEXT: v_mov_b32_e32 v0, s12403; GFX9-NEXT: v_mov_b32_e32 v1, s13404; GFX9-NEXT: s_addc_u32 s1, s13, s15405; GFX9-NEXT: v_cmp_lt_i64_e64 s[2:3], s[14:15], 0406; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]407; GFX9-NEXT: v_mov_b32_e32 v0, s0408; GFX9-NEXT: v_mov_b32_e32 v1, s1409; GFX9-NEXT: s_xor_b64 s[0:1], s[2:3], vcc410; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]411; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]412; GFX9-NEXT: global_store_byte v2, v0, s[10:11]413; GFX9-NEXT: s_endpgm414;415; GFX10-LABEL: s_saddo_i64:416; GFX10: ; %bb.0:417; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24418; GFX10-NEXT: v_mov_b32_e32 v2, 0419; GFX10-NEXT: s_waitcnt lgkmcnt(0)420; GFX10-NEXT: s_add_u32 s0, s12, s14421; GFX10-NEXT: s_addc_u32 s1, s13, s15422; GFX10-NEXT: v_cmp_lt_i64_e64 s2, s[14:15], 0423; GFX10-NEXT: v_cmp_lt_i64_e64 s3, s[0:1], s[12:13]424; GFX10-NEXT: v_mov_b32_e32 v0, s0425; GFX10-NEXT: v_mov_b32_e32 v1, s1426; GFX10-NEXT: s_xor_b32 s0, s2, s3427; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0428; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]429; GFX10-NEXT: global_store_byte v2, v3, s[10:11]430; GFX10-NEXT: s_endpgm431;432; GFX11-LABEL: s_saddo_i64:433; GFX11: ; %bb.0:434; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24435; GFX11-NEXT: s_waitcnt lgkmcnt(0)436; GFX11-NEXT: s_add_u32 s8, s4, s6437; GFX11-NEXT: s_addc_u32 s9, s5, s7438; GFX11-NEXT: v_cmp_lt_i64_e64 s6, s[6:7], 0439; GFX11-NEXT: v_cmp_lt_i64_e64 s4, s[8:9], s[4:5]440; GFX11-NEXT: v_mov_b32_e32 v0, s8441; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s9442; GFX11-NEXT: s_xor_b32 s4, s6, s4443; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)444; GFX11-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4445; GFX11-NEXT: s_clause 0x1446; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]447; GFX11-NEXT: global_store_b8 v2, v3, s[2:3]448; GFX11-NEXT: s_endpgm449 %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind450 %val = extractvalue { i64, i1 } %sadd, 0451 %carry = extractvalue { i64, i1 } %sadd, 1452 store i64 %val, ptr addrspace(1) %out, align 8453 store i1 %carry, ptr addrspace(1) %carryout454 ret void455}456 457define amdgpu_kernel void @v_saddo_i64(ptr addrspace(1) %out, ptr addrspace(1) %carryout, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) nounwind {458; SI-LABEL: v_saddo_i64:459; SI: ; %bb.0:460; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9461; SI-NEXT: s_mov_b32 s11, 0xf000462; SI-NEXT: s_mov_b32 s10, -1463; SI-NEXT: s_mov_b32 s14, s10464; SI-NEXT: s_mov_b32 s15, s11465; SI-NEXT: s_waitcnt lgkmcnt(0)466; SI-NEXT: s_mov_b32 s12, s4467; SI-NEXT: s_mov_b32 s13, s5468; SI-NEXT: s_mov_b32 s4, s6469; SI-NEXT: s_mov_b32 s5, s7470; SI-NEXT: s_mov_b32 s6, s10471; SI-NEXT: s_mov_b32 s7, s11472; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[12:15], 0473; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0474; SI-NEXT: s_mov_b32 s8, s0475; SI-NEXT: s_mov_b32 s9, s1476; SI-NEXT: s_mov_b32 s4, s2477; SI-NEXT: s_mov_b32 s5, s3478; SI-NEXT: s_waitcnt vmcnt(0)479; SI-NEXT: v_add_i32_e32 v4, vcc, v0, v2480; SI-NEXT: v_addc_u32_e32 v5, vcc, v1, v3, vcc481; SI-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3]482; SI-NEXT: v_cmp_lt_i64_e64 s[0:1], v[4:5], v[0:1]483; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[8:11], 0484; SI-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]485; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]486; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0487; SI-NEXT: s_endpgm488;489; VI-LABEL: v_saddo_i64:490; VI: ; %bb.0:491; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24492; VI-NEXT: s_waitcnt lgkmcnt(0)493; VI-NEXT: v_mov_b32_e32 v0, s4494; VI-NEXT: v_mov_b32_e32 v1, s5495; VI-NEXT: v_mov_b32_e32 v2, s6496; VI-NEXT: v_mov_b32_e32 v3, s7497; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]498; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]499; VI-NEXT: v_mov_b32_e32 v4, s0500; VI-NEXT: v_mov_b32_e32 v5, s1501; VI-NEXT: v_mov_b32_e32 v6, s2502; VI-NEXT: v_mov_b32_e32 v7, s3503; VI-NEXT: s_waitcnt vmcnt(0)504; VI-NEXT: v_add_u32_e32 v8, vcc, v0, v2505; VI-NEXT: v_addc_u32_e32 v9, vcc, v1, v3, vcc506; VI-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3]507; VI-NEXT: v_cmp_lt_i64_e64 s[0:1], v[8:9], v[0:1]508; VI-NEXT: flat_store_dwordx2 v[4:5], v[8:9]509; VI-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]510; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]511; VI-NEXT: flat_store_byte v[6:7], v0512; VI-NEXT: s_endpgm513;514; GFX9-LABEL: v_saddo_i64:515; GFX9: ; %bb.0:516; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24517; GFX9-NEXT: v_mov_b32_e32 v6, 0518; GFX9-NEXT: s_waitcnt lgkmcnt(0)519; GFX9-NEXT: global_load_dwordx2 v[0:1], v6, s[12:13]520; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[14:15]521; GFX9-NEXT: s_waitcnt vmcnt(0)522; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v0, v2523; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v3, vcc524; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[2:3]525; GFX9-NEXT: v_cmp_lt_i64_e64 s[0:1], v[4:5], v[0:1]526; GFX9-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]527; GFX9-NEXT: s_xor_b64 s[0:1], vcc, s[0:1]528; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]529; GFX9-NEXT: global_store_byte v6, v0, s[10:11]530; GFX9-NEXT: s_endpgm531;532; GFX10-LABEL: v_saddo_i64:533; GFX10: ; %bb.0:534; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24535; GFX10-NEXT: v_mov_b32_e32 v6, 0536; GFX10-NEXT: s_waitcnt lgkmcnt(0)537; GFX10-NEXT: s_clause 0x1538; GFX10-NEXT: global_load_dwordx2 v[0:1], v6, s[12:13]539; GFX10-NEXT: global_load_dwordx2 v[2:3], v6, s[14:15]540; GFX10-NEXT: s_waitcnt vmcnt(0)541; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2542; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo543; GFX10-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]544; GFX10-NEXT: v_cmp_lt_i64_e64 s0, v[4:5], v[0:1]545; GFX10-NEXT: s_xor_b32 s0, vcc_lo, s0546; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0547; GFX10-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]548; GFX10-NEXT: global_store_byte v6, v0, s[10:11]549; GFX10-NEXT: s_endpgm550;551; GFX11-LABEL: v_saddo_i64:552; GFX11: ; %bb.0:553; GFX11-NEXT: s_load_b256 s[4:11], s[4:5], 0x24554; GFX11-NEXT: v_mov_b32_e32 v6, 0555; GFX11-NEXT: s_waitcnt lgkmcnt(0)556; GFX11-NEXT: s_clause 0x1557; GFX11-NEXT: global_load_b64 v[0:1], v6, s[8:9]558; GFX11-NEXT: global_load_b64 v[2:3], v6, s[10:11]559; GFX11-NEXT: s_waitcnt vmcnt(0)560; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2561; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)562; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo563; GFX11-NEXT: v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]564; GFX11-NEXT: v_cmp_lt_i64_e64 s0, v[4:5], v[0:1]565; GFX11-NEXT: s_xor_b32 s0, vcc_lo, s0566; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)567; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0568; GFX11-NEXT: s_clause 0x1569; GFX11-NEXT: global_store_b64 v6, v[4:5], s[4:5]570; GFX11-NEXT: global_store_b8 v6, v0, s[6:7]571; GFX11-NEXT: s_endpgm572 %a = load i64, ptr addrspace(1) %aptr, align 4573 %b = load i64, ptr addrspace(1) %bptr, align 4574 %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind575 %val = extractvalue { i64, i1 } %sadd, 0576 %carry = extractvalue { i64, i1 } %sadd, 1577 store i64 %val, ptr addrspace(1) %out, align 8578 store i1 %carry, ptr addrspace(1) %carryout579 ret void580}581 582define amdgpu_kernel void @v_saddo_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %carryout, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) nounwind {583; SI-LABEL: v_saddo_v2i32:584; SI: ; %bb.0:585; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9586; SI-NEXT: s_mov_b32 s11, 0xf000587; SI-NEXT: s_mov_b32 s10, -1588; SI-NEXT: s_mov_b32 s14, s10589; SI-NEXT: s_mov_b32 s15, s11590; SI-NEXT: s_waitcnt lgkmcnt(0)591; SI-NEXT: s_mov_b32 s12, s4592; SI-NEXT: s_mov_b32 s13, s5593; SI-NEXT: s_mov_b32 s4, s6594; SI-NEXT: s_mov_b32 s5, s7595; SI-NEXT: s_mov_b32 s6, s10596; SI-NEXT: s_mov_b32 s7, s11597; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[12:15], 0598; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0599; SI-NEXT: s_mov_b32 s8, s0600; SI-NEXT: s_mov_b32 s9, s1601; SI-NEXT: s_mov_b32 s12, s2602; SI-NEXT: s_mov_b32 s13, s3603; SI-NEXT: s_waitcnt vmcnt(0)604; SI-NEXT: v_add_i32_e32 v5, vcc, v1, v3605; SI-NEXT: v_add_i32_e32 v4, vcc, v0, v2606; SI-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v3607; SI-NEXT: v_cmp_lt_i32_e64 s[4:5], v5, v1608; SI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2609; SI-NEXT: v_cmp_lt_i32_e64 s[2:3], v4, v0610; SI-NEXT: s_xor_b64 s[0:1], s[0:1], s[4:5]611; SI-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]612; SI-NEXT: s_xor_b64 s[0:1], vcc, s[2:3]613; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]614; SI-NEXT: buffer_store_dwordx2 v[4:5], off, s[8:11], 0615; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[12:15], 0616; SI-NEXT: s_endpgm617;618; VI-LABEL: v_saddo_v2i32:619; VI: ; %bb.0:620; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24621; VI-NEXT: s_waitcnt lgkmcnt(0)622; VI-NEXT: v_mov_b32_e32 v0, s4623; VI-NEXT: v_mov_b32_e32 v1, s5624; VI-NEXT: v_mov_b32_e32 v2, s6625; VI-NEXT: v_mov_b32_e32 v3, s7626; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]627; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]628; VI-NEXT: v_mov_b32_e32 v4, s0629; VI-NEXT: v_mov_b32_e32 v5, s1630; VI-NEXT: v_mov_b32_e32 v6, s2631; VI-NEXT: v_mov_b32_e32 v7, s3632; VI-NEXT: s_waitcnt vmcnt(0)633; VI-NEXT: v_add_u32_e32 v9, vcc, v1, v3634; VI-NEXT: v_add_u32_e32 v8, vcc, v0, v2635; VI-NEXT: v_cmp_gt_i32_e64 s[0:1], 0, v3636; VI-NEXT: v_cmp_lt_i32_e64 s[4:5], v9, v1637; VI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2638; VI-NEXT: v_cmp_lt_i32_e64 s[2:3], v8, v0639; VI-NEXT: s_xor_b64 s[0:1], s[0:1], s[4:5]640; VI-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]641; VI-NEXT: s_xor_b64 s[0:1], vcc, s[2:3]642; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]643; VI-NEXT: flat_store_dwordx2 v[4:5], v[8:9]644; VI-NEXT: flat_store_dwordx2 v[6:7], v[0:1]645; VI-NEXT: s_endpgm646;647; GFX9-LABEL: v_saddo_v2i32:648; GFX9: ; %bb.0:649; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24650; GFX9-NEXT: v_mov_b32_e32 v6, 0651; GFX9-NEXT: s_waitcnt lgkmcnt(0)652; GFX9-NEXT: global_load_dwordx2 v[0:1], v6, s[12:13]653; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[14:15]654; GFX9-NEXT: s_waitcnt vmcnt(0)655; GFX9-NEXT: v_add_u32_e32 v5, v1, v3656; GFX9-NEXT: v_add_i32 v1, v1, v3 clamp657; GFX9-NEXT: v_add_u32_e32 v4, v0, v2658; GFX9-NEXT: v_add_i32 v0, v0, v2 clamp659; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v5, v1660; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc661; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v4, v0662; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc663; GFX9-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]664; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[10:11]665; GFX9-NEXT: s_endpgm666;667; GFX10-LABEL: v_saddo_v2i32:668; GFX10: ; %bb.0:669; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24670; GFX10-NEXT: v_mov_b32_e32 v5, 0671; GFX10-NEXT: s_waitcnt lgkmcnt(0)672; GFX10-NEXT: s_clause 0x1673; GFX10-NEXT: global_load_dwordx2 v[0:1], v5, s[12:13]674; GFX10-NEXT: global_load_dwordx2 v[2:3], v5, s[14:15]675; GFX10-NEXT: s_waitcnt vmcnt(0)676; GFX10-NEXT: v_add_nc_u32_e32 v4, v1, v3677; GFX10-NEXT: v_add_nc_i32 v1, v1, v3 clamp678; GFX10-NEXT: v_add_nc_u32_e32 v3, v0, v2679; GFX10-NEXT: v_add_nc_i32 v0, v0, v2 clamp680; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v1681; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo682; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v0683; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo684; GFX10-NEXT: global_store_dwordx2 v5, v[3:4], s[8:9]685; GFX10-NEXT: global_store_dwordx2 v5, v[0:1], s[10:11]686; GFX10-NEXT: s_endpgm687;688; GFX11-LABEL: v_saddo_v2i32:689; GFX11: ; %bb.0:690; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24691; GFX11-NEXT: v_mov_b32_e32 v5, 0692; GFX11-NEXT: s_waitcnt lgkmcnt(0)693; GFX11-NEXT: s_clause 0x1694; GFX11-NEXT: global_load_b64 v[0:1], v5, s[4:5]695; GFX11-NEXT: global_load_b64 v[2:3], v5, s[6:7]696; GFX11-NEXT: s_waitcnt vmcnt(0)697; GFX11-NEXT: v_add_nc_u32_e32 v4, v1, v3698; GFX11-NEXT: v_add_nc_i32 v1, v1, v3 clamp699; GFX11-NEXT: v_add_nc_u32_e32 v3, v0, v2700; GFX11-NEXT: v_add_nc_i32 v0, v0, v2 clamp701; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)702; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v1703; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo704; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v0705; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo706; GFX11-NEXT: s_clause 0x1707; GFX11-NEXT: global_store_b64 v5, v[3:4], s[0:1]708; GFX11-NEXT: global_store_b64 v5, v[0:1], s[2:3]709; GFX11-NEXT: s_endpgm710 %a = load <2 x i32>, ptr addrspace(1) %aptr, align 4711 %b = load <2 x i32>, ptr addrspace(1) %bptr, align 4712 %sadd = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind713 %val = extractvalue { <2 x i32>, <2 x i1> } %sadd, 0714 %carry = extractvalue { <2 x i32>, <2 x i1> } %sadd, 1715 store <2 x i32> %val, ptr addrspace(1) %out, align 4716 %carry.ext = zext <2 x i1> %carry to <2 x i32>717 store <2 x i32> %carry.ext, ptr addrspace(1) %carryout718 ret void719}720