626 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri < %s | FileCheck -check-prefixes=GCN,CI %s3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s8 9define i8 @atomic_load_monotonic_i8(ptr addrspace(3) %ptr) {10; CI-LABEL: atomic_load_monotonic_i8:11; CI: ; %bb.0:12; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13; CI-NEXT: s_mov_b32 m0, -114; CI-NEXT: ds_read_u8 v0, v015; CI-NEXT: s_waitcnt lgkmcnt(0)16; CI-NEXT: s_setpc_b64 s[30:31]17;18; GFX9-LABEL: atomic_load_monotonic_i8:19; GFX9: ; %bb.0:20; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21; GFX9-NEXT: ds_read_u8 v0, v022; GFX9-NEXT: s_waitcnt lgkmcnt(0)23; GFX9-NEXT: s_setpc_b64 s[30:31]24;25; GFX11-TRUE16-LABEL: atomic_load_monotonic_i8:26; GFX11-TRUE16: ; %bb.0:27; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)28; GFX11-TRUE16-NEXT: ds_load_u8_d16 v0, v029; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)30; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]31;32; GFX11-FAKE16-LABEL: atomic_load_monotonic_i8:33; GFX11-FAKE16: ; %bb.0:34; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)35; GFX11-FAKE16-NEXT: ds_load_u8 v0, v036; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)37; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]38;39; GFX1250-LABEL: atomic_load_monotonic_i8:40; GFX1250: ; %bb.0:41; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x042; GFX1250-NEXT: s_wait_kmcnt 0x043; GFX1250-NEXT: ds_load_u8 v0, v044; GFX1250-NEXT: s_wait_dscnt 0x045; GFX1250-NEXT: s_set_pc_i64 s[30:31]46 %load = load atomic i8, ptr addrspace(3) %ptr monotonic, align 147 ret i8 %load48}49 50define i8 @atomic_load_monotonic_i8_offset(ptr addrspace(3) %ptr) {51; CI-LABEL: atomic_load_monotonic_i8_offset:52; CI: ; %bb.0:53; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)54; CI-NEXT: s_mov_b32 m0, -155; CI-NEXT: ds_read_u8 v0, v0 offset:1656; CI-NEXT: s_waitcnt lgkmcnt(0)57; CI-NEXT: s_setpc_b64 s[30:31]58;59; GFX9-LABEL: atomic_load_monotonic_i8_offset:60; GFX9: ; %bb.0:61; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)62; GFX9-NEXT: ds_read_u8 v0, v0 offset:1663; GFX9-NEXT: s_waitcnt lgkmcnt(0)64; GFX9-NEXT: s_setpc_b64 s[30:31]65;66; GFX11-TRUE16-LABEL: atomic_load_monotonic_i8_offset:67; GFX11-TRUE16: ; %bb.0:68; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)69; GFX11-TRUE16-NEXT: ds_load_u8_d16 v0, v0 offset:1670; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)71; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]72;73; GFX11-FAKE16-LABEL: atomic_load_monotonic_i8_offset:74; GFX11-FAKE16: ; %bb.0:75; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)76; GFX11-FAKE16-NEXT: ds_load_u8 v0, v0 offset:1677; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)78; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]79;80; GFX1250-LABEL: atomic_load_monotonic_i8_offset:81; GFX1250: ; %bb.0:82; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x083; GFX1250-NEXT: s_wait_kmcnt 0x084; GFX1250-NEXT: ds_load_u8 v0, v0 offset:1685; GFX1250-NEXT: s_wait_dscnt 0x086; GFX1250-NEXT: s_set_pc_i64 s[30:31]87 %gep = getelementptr inbounds i8, ptr addrspace(3) %ptr, i8 1688 %load = load atomic i8, ptr addrspace(3) %gep monotonic, align 189 ret i8 %load90}91 92define i16 @atomic_load_monotonic_i16(ptr addrspace(3) %ptr) {93; CI-LABEL: atomic_load_monotonic_i16:94; CI: ; %bb.0:95; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)96; CI-NEXT: s_mov_b32 m0, -197; CI-NEXT: ds_read_u16 v0, v098; CI-NEXT: s_waitcnt lgkmcnt(0)99; CI-NEXT: s_setpc_b64 s[30:31]100;101; GFX9-LABEL: atomic_load_monotonic_i16:102; GFX9: ; %bb.0:103; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)104; GFX9-NEXT: ds_read_u16 v0, v0105; GFX9-NEXT: s_waitcnt lgkmcnt(0)106; GFX9-NEXT: s_setpc_b64 s[30:31]107;108; GFX11-TRUE16-LABEL: atomic_load_monotonic_i16:109; GFX11-TRUE16: ; %bb.0:110; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)111; GFX11-TRUE16-NEXT: ds_load_u16_d16 v0, v0112; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)113; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]114;115; GFX11-FAKE16-LABEL: atomic_load_monotonic_i16:116; GFX11-FAKE16: ; %bb.0:117; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)118; GFX11-FAKE16-NEXT: ds_load_u16 v0, v0119; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)120; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]121;122; GFX1250-LABEL: atomic_load_monotonic_i16:123; GFX1250: ; %bb.0:124; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0125; GFX1250-NEXT: s_wait_kmcnt 0x0126; GFX1250-NEXT: ds_load_u16 v0, v0127; GFX1250-NEXT: s_wait_dscnt 0x0128; GFX1250-NEXT: s_set_pc_i64 s[30:31]129 %load = load atomic i16, ptr addrspace(3) %ptr monotonic, align 2130 ret i16 %load131}132 133define i16 @atomic_load_monotonic_i16_offset(ptr addrspace(3) %ptr) {134; CI-LABEL: atomic_load_monotonic_i16_offset:135; CI: ; %bb.0:136; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)137; CI-NEXT: s_mov_b32 m0, -1138; CI-NEXT: ds_read_u16 v0, v0 offset:32139; CI-NEXT: s_waitcnt lgkmcnt(0)140; CI-NEXT: s_setpc_b64 s[30:31]141;142; GFX9-LABEL: atomic_load_monotonic_i16_offset:143; GFX9: ; %bb.0:144; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)145; GFX9-NEXT: ds_read_u16 v0, v0 offset:32146; GFX9-NEXT: s_waitcnt lgkmcnt(0)147; GFX9-NEXT: s_setpc_b64 s[30:31]148;149; GFX11-TRUE16-LABEL: atomic_load_monotonic_i16_offset:150; GFX11-TRUE16: ; %bb.0:151; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)152; GFX11-TRUE16-NEXT: ds_load_u16_d16 v0, v0 offset:32153; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)154; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]155;156; GFX11-FAKE16-LABEL: atomic_load_monotonic_i16_offset:157; GFX11-FAKE16: ; %bb.0:158; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)159; GFX11-FAKE16-NEXT: ds_load_u16 v0, v0 offset:32160; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)161; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]162;163; GFX1250-LABEL: atomic_load_monotonic_i16_offset:164; GFX1250: ; %bb.0:165; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0166; GFX1250-NEXT: s_wait_kmcnt 0x0167; GFX1250-NEXT: ds_load_u16 v0, v0 offset:32168; GFX1250-NEXT: s_wait_dscnt 0x0169; GFX1250-NEXT: s_set_pc_i64 s[30:31]170 %gep = getelementptr inbounds i16, ptr addrspace(3) %ptr, i16 16171 %load = load atomic i16, ptr addrspace(3) %gep monotonic, align 2172 ret i16 %load173}174 175define i32 @atomic_load_monotonic_i32(ptr addrspace(3) %ptr) {176; CI-LABEL: atomic_load_monotonic_i32:177; CI: ; %bb.0:178; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)179; CI-NEXT: s_mov_b32 m0, -1180; CI-NEXT: ds_read_b32 v0, v0181; CI-NEXT: s_waitcnt lgkmcnt(0)182; CI-NEXT: s_setpc_b64 s[30:31]183;184; GFX9-LABEL: atomic_load_monotonic_i32:185; GFX9: ; %bb.0:186; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)187; GFX9-NEXT: ds_read_b32 v0, v0188; GFX9-NEXT: s_waitcnt lgkmcnt(0)189; GFX9-NEXT: s_setpc_b64 s[30:31]190;191; GFX11-LABEL: atomic_load_monotonic_i32:192; GFX11: ; %bb.0:193; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)194; GFX11-NEXT: ds_load_b32 v0, v0195; GFX11-NEXT: s_waitcnt lgkmcnt(0)196; GFX11-NEXT: s_setpc_b64 s[30:31]197;198; GFX1250-LABEL: atomic_load_monotonic_i32:199; GFX1250: ; %bb.0:200; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0201; GFX1250-NEXT: s_wait_kmcnt 0x0202; GFX1250-NEXT: ds_load_b32 v0, v0203; GFX1250-NEXT: s_wait_dscnt 0x0204; GFX1250-NEXT: s_set_pc_i64 s[30:31]205 %load = load atomic i32, ptr addrspace(3) %ptr monotonic, align 4206 ret i32 %load207}208 209define i32 @atomic_load_monotonic_i32_offset(ptr addrspace(3) %ptr) {210; CI-LABEL: atomic_load_monotonic_i32_offset:211; CI: ; %bb.0:212; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)213; CI-NEXT: s_mov_b32 m0, -1214; CI-NEXT: ds_read_b32 v0, v0 offset:64215; CI-NEXT: s_waitcnt lgkmcnt(0)216; CI-NEXT: s_setpc_b64 s[30:31]217;218; GFX9-LABEL: atomic_load_monotonic_i32_offset:219; GFX9: ; %bb.0:220; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)221; GFX9-NEXT: ds_read_b32 v0, v0 offset:64222; GFX9-NEXT: s_waitcnt lgkmcnt(0)223; GFX9-NEXT: s_setpc_b64 s[30:31]224;225; GFX11-LABEL: atomic_load_monotonic_i32_offset:226; GFX11: ; %bb.0:227; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)228; GFX11-NEXT: ds_load_b32 v0, v0 offset:64229; GFX11-NEXT: s_waitcnt lgkmcnt(0)230; GFX11-NEXT: s_setpc_b64 s[30:31]231;232; GFX1250-LABEL: atomic_load_monotonic_i32_offset:233; GFX1250: ; %bb.0:234; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0235; GFX1250-NEXT: s_wait_kmcnt 0x0236; GFX1250-NEXT: ds_load_b32 v0, v0 offset:64237; GFX1250-NEXT: s_wait_dscnt 0x0238; GFX1250-NEXT: s_set_pc_i64 s[30:31]239 %gep = getelementptr inbounds i32, ptr addrspace(3) %ptr, i32 16240 %load = load atomic i32, ptr addrspace(3) %gep monotonic, align 4241 ret i32 %load242}243 244define i64 @atomic_load_monotonic_i64(ptr addrspace(3) %ptr) {245; CI-LABEL: atomic_load_monotonic_i64:246; CI: ; %bb.0:247; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)248; CI-NEXT: s_mov_b32 m0, -1249; CI-NEXT: ds_read_b64 v[0:1], v0250; CI-NEXT: s_waitcnt lgkmcnt(0)251; CI-NEXT: s_setpc_b64 s[30:31]252;253; GFX9-LABEL: atomic_load_monotonic_i64:254; GFX9: ; %bb.0:255; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)256; GFX9-NEXT: ds_read_b64 v[0:1], v0257; GFX9-NEXT: s_waitcnt lgkmcnt(0)258; GFX9-NEXT: s_setpc_b64 s[30:31]259;260; GFX11-LABEL: atomic_load_monotonic_i64:261; GFX11: ; %bb.0:262; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)263; GFX11-NEXT: ds_load_b64 v[0:1], v0264; GFX11-NEXT: s_waitcnt lgkmcnt(0)265; GFX11-NEXT: s_setpc_b64 s[30:31]266;267; GFX1250-LABEL: atomic_load_monotonic_i64:268; GFX1250: ; %bb.0:269; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0270; GFX1250-NEXT: s_wait_kmcnt 0x0271; GFX1250-NEXT: ds_load_b64 v[0:1], v0272; GFX1250-NEXT: s_wait_dscnt 0x0273; GFX1250-NEXT: s_set_pc_i64 s[30:31]274 %load = load atomic i64, ptr addrspace(3) %ptr monotonic, align 8275 ret i64 %load276}277 278define i64 @atomic_load_monotonic_i64_offset(ptr addrspace(3) %ptr) {279; CI-LABEL: atomic_load_monotonic_i64_offset:280; CI: ; %bb.0:281; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)282; CI-NEXT: s_mov_b32 m0, -1283; CI-NEXT: ds_read_b64 v[0:1], v0 offset:128284; CI-NEXT: s_waitcnt lgkmcnt(0)285; CI-NEXT: s_setpc_b64 s[30:31]286;287; GFX9-LABEL: atomic_load_monotonic_i64_offset:288; GFX9: ; %bb.0:289; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)290; GFX9-NEXT: ds_read_b64 v[0:1], v0 offset:128291; GFX9-NEXT: s_waitcnt lgkmcnt(0)292; GFX9-NEXT: s_setpc_b64 s[30:31]293;294; GFX11-LABEL: atomic_load_monotonic_i64_offset:295; GFX11: ; %bb.0:296; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)297; GFX11-NEXT: ds_load_b64 v[0:1], v0 offset:128298; GFX11-NEXT: s_waitcnt lgkmcnt(0)299; GFX11-NEXT: s_setpc_b64 s[30:31]300;301; GFX1250-LABEL: atomic_load_monotonic_i64_offset:302; GFX1250: ; %bb.0:303; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0304; GFX1250-NEXT: s_wait_kmcnt 0x0305; GFX1250-NEXT: ds_load_b64 v[0:1], v0 offset:128306; GFX1250-NEXT: s_wait_dscnt 0x0307; GFX1250-NEXT: s_set_pc_i64 s[30:31]308 %gep = getelementptr inbounds i64, ptr addrspace(3) %ptr, i32 16309 %load = load atomic i64, ptr addrspace(3) %gep monotonic, align 8310 ret i64 %load311}312 313define float @atomic_load_monotonic_f32_offset(ptr addrspace(3) %ptr) {314; CI-LABEL: atomic_load_monotonic_f32_offset:315; CI: ; %bb.0:316; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)317; CI-NEXT: s_mov_b32 m0, -1318; CI-NEXT: ds_read_b32 v0, v0 offset:64319; CI-NEXT: s_waitcnt lgkmcnt(0)320; CI-NEXT: s_setpc_b64 s[30:31]321;322; GFX9-LABEL: atomic_load_monotonic_f32_offset:323; GFX9: ; %bb.0:324; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)325; GFX9-NEXT: ds_read_b32 v0, v0 offset:64326; GFX9-NEXT: s_waitcnt lgkmcnt(0)327; GFX9-NEXT: s_setpc_b64 s[30:31]328;329; GFX11-LABEL: atomic_load_monotonic_f32_offset:330; GFX11: ; %bb.0:331; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)332; GFX11-NEXT: ds_load_b32 v0, v0 offset:64333; GFX11-NEXT: s_waitcnt lgkmcnt(0)334; GFX11-NEXT: s_setpc_b64 s[30:31]335;336; GFX1250-LABEL: atomic_load_monotonic_f32_offset:337; GFX1250: ; %bb.0:338; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0339; GFX1250-NEXT: s_wait_kmcnt 0x0340; GFX1250-NEXT: ds_load_b32 v0, v0 offset:64341; GFX1250-NEXT: s_wait_dscnt 0x0342; GFX1250-NEXT: s_set_pc_i64 s[30:31]343 %gep = getelementptr inbounds float, ptr addrspace(3) %ptr, i32 16344 %load = load atomic float, ptr addrspace(3) %gep monotonic, align 4345 ret float %load346}347 348define double @atomic_load_monotonic_f64_offset(ptr addrspace(3) %ptr) {349; CI-LABEL: atomic_load_monotonic_f64_offset:350; CI: ; %bb.0:351; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)352; CI-NEXT: s_mov_b32 m0, -1353; CI-NEXT: ds_read_b64 v[0:1], v0 offset:128354; CI-NEXT: s_waitcnt lgkmcnt(0)355; CI-NEXT: s_setpc_b64 s[30:31]356;357; GFX9-LABEL: atomic_load_monotonic_f64_offset:358; GFX9: ; %bb.0:359; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)360; GFX9-NEXT: ds_read_b64 v[0:1], v0 offset:128361; GFX9-NEXT: s_waitcnt lgkmcnt(0)362; GFX9-NEXT: s_setpc_b64 s[30:31]363;364; GFX11-LABEL: atomic_load_monotonic_f64_offset:365; GFX11: ; %bb.0:366; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)367; GFX11-NEXT: ds_load_b64 v[0:1], v0 offset:128368; GFX11-NEXT: s_waitcnt lgkmcnt(0)369; GFX11-NEXT: s_setpc_b64 s[30:31]370;371; GFX1250-LABEL: atomic_load_monotonic_f64_offset:372; GFX1250: ; %bb.0:373; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0374; GFX1250-NEXT: s_wait_kmcnt 0x0375; GFX1250-NEXT: ds_load_b64 v[0:1], v0 offset:128376; GFX1250-NEXT: s_wait_dscnt 0x0377; GFX1250-NEXT: s_set_pc_i64 s[30:31]378 %gep = getelementptr inbounds double, ptr addrspace(3) %ptr, i32 16379 %load = load atomic double, ptr addrspace(3) %gep monotonic, align 8380 ret double %load381}382 383define ptr @atomic_load_monotonic_p0i8_offset(ptr addrspace(3) %ptr) {384; CI-LABEL: atomic_load_monotonic_p0i8_offset:385; CI: ; %bb.0:386; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)387; CI-NEXT: s_mov_b32 m0, -1388; CI-NEXT: ds_read_b64 v[0:1], v0 offset:128389; CI-NEXT: s_waitcnt lgkmcnt(0)390; CI-NEXT: s_setpc_b64 s[30:31]391;392; GFX9-LABEL: atomic_load_monotonic_p0i8_offset:393; GFX9: ; %bb.0:394; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)395; GFX9-NEXT: ds_read_b64 v[0:1], v0 offset:128396; GFX9-NEXT: s_waitcnt lgkmcnt(0)397; GFX9-NEXT: s_setpc_b64 s[30:31]398;399; GFX11-LABEL: atomic_load_monotonic_p0i8_offset:400; GFX11: ; %bb.0:401; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)402; GFX11-NEXT: ds_load_b64 v[0:1], v0 offset:128403; GFX11-NEXT: s_waitcnt lgkmcnt(0)404; GFX11-NEXT: s_setpc_b64 s[30:31]405;406; GFX1250-LABEL: atomic_load_monotonic_p0i8_offset:407; GFX1250: ; %bb.0:408; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0409; GFX1250-NEXT: s_wait_kmcnt 0x0410; GFX1250-NEXT: ds_load_b64 v[0:1], v0 offset:128411; GFX1250-NEXT: s_wait_dscnt 0x0412; GFX1250-NEXT: s_set_pc_i64 s[30:31]413 %gep = getelementptr inbounds ptr, ptr addrspace(3) %ptr, i32 16414 %load = load atomic ptr, ptr addrspace(3) %gep monotonic, align 8415 ret ptr %load416}417 418define ptr addrspace(3) @atomic_load_monotonic_p3i8_offset(ptr addrspace(3) %ptr) {419; CI-LABEL: atomic_load_monotonic_p3i8_offset:420; CI: ; %bb.0:421; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)422; CI-NEXT: s_mov_b32 m0, -1423; CI-NEXT: ds_read_b32 v0, v0 offset:64424; CI-NEXT: s_waitcnt lgkmcnt(0)425; CI-NEXT: s_setpc_b64 s[30:31]426;427; GFX9-LABEL: atomic_load_monotonic_p3i8_offset:428; GFX9: ; %bb.0:429; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)430; GFX9-NEXT: ds_read_b32 v0, v0 offset:64431; GFX9-NEXT: s_waitcnt lgkmcnt(0)432; GFX9-NEXT: s_setpc_b64 s[30:31]433;434; GFX11-LABEL: atomic_load_monotonic_p3i8_offset:435; GFX11: ; %bb.0:436; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)437; GFX11-NEXT: ds_load_b32 v0, v0 offset:64438; GFX11-NEXT: s_waitcnt lgkmcnt(0)439; GFX11-NEXT: s_setpc_b64 s[30:31]440;441; GFX1250-LABEL: atomic_load_monotonic_p3i8_offset:442; GFX1250: ; %bb.0:443; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0444; GFX1250-NEXT: s_wait_kmcnt 0x0445; GFX1250-NEXT: ds_load_b32 v0, v0 offset:64446; GFX1250-NEXT: s_wait_dscnt 0x0447; GFX1250-NEXT: s_set_pc_i64 s[30:31]448 %gep = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %ptr, i32 16449 %load = load atomic ptr addrspace(3), ptr addrspace(3) %gep monotonic, align 4450 ret ptr addrspace(3) %load451}452 453define i16 @atomic_load_monotonic_f16(ptr addrspace(3) %ptr) {454; CI-LABEL: atomic_load_monotonic_f16:455; CI: ; %bb.0:456; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)457; CI-NEXT: s_mov_b32 m0, -1458; CI-NEXT: ds_read_u16 v0, v0459; CI-NEXT: s_waitcnt lgkmcnt(0)460; CI-NEXT: s_setpc_b64 s[30:31]461;462; GFX9-LABEL: atomic_load_monotonic_f16:463; GFX9: ; %bb.0:464; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)465; GFX9-NEXT: ds_read_u16 v0, v0466; GFX9-NEXT: s_waitcnt lgkmcnt(0)467; GFX9-NEXT: s_setpc_b64 s[30:31]468;469; GFX11-TRUE16-LABEL: atomic_load_monotonic_f16:470; GFX11-TRUE16: ; %bb.0:471; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)472; GFX11-TRUE16-NEXT: ds_load_u16_d16 v0, v0473; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)474; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]475;476; GFX11-FAKE16-LABEL: atomic_load_monotonic_f16:477; GFX11-FAKE16: ; %bb.0:478; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)479; GFX11-FAKE16-NEXT: ds_load_u16 v0, v0480; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)481; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]482;483; GFX1250-LABEL: atomic_load_monotonic_f16:484; GFX1250: ; %bb.0:485; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0486; GFX1250-NEXT: s_wait_kmcnt 0x0487; GFX1250-NEXT: ds_load_u16 v0, v0488; GFX1250-NEXT: s_wait_dscnt 0x0489; GFX1250-NEXT: s_set_pc_i64 s[30:31]490 %load = load atomic half, ptr addrspace(3) %ptr monotonic, align 2491 %ret = bitcast half %load to i16492 ret i16 %ret493}494 495define i16 @atomic_load_monotonic_f16_offset(ptr addrspace(3) %ptr) {496; CI-LABEL: atomic_load_monotonic_f16_offset:497; CI: ; %bb.0:498; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)499; CI-NEXT: s_mov_b32 m0, -1500; CI-NEXT: ds_read_u16 v0, v0 offset:32501; CI-NEXT: s_waitcnt lgkmcnt(0)502; CI-NEXT: s_setpc_b64 s[30:31]503;504; GFX9-LABEL: atomic_load_monotonic_f16_offset:505; GFX9: ; %bb.0:506; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)507; GFX9-NEXT: ds_read_u16 v0, v0 offset:32508; GFX9-NEXT: s_waitcnt lgkmcnt(0)509; GFX9-NEXT: s_setpc_b64 s[30:31]510;511; GFX11-TRUE16-LABEL: atomic_load_monotonic_f16_offset:512; GFX11-TRUE16: ; %bb.0:513; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)514; GFX11-TRUE16-NEXT: ds_load_u16_d16 v0, v0 offset:32515; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)516; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]517;518; GFX11-FAKE16-LABEL: atomic_load_monotonic_f16_offset:519; GFX11-FAKE16: ; %bb.0:520; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)521; GFX11-FAKE16-NEXT: ds_load_u16 v0, v0 offset:32522; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)523; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]524;525; GFX1250-LABEL: atomic_load_monotonic_f16_offset:526; GFX1250: ; %bb.0:527; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0528; GFX1250-NEXT: s_wait_kmcnt 0x0529; GFX1250-NEXT: ds_load_u16 v0, v0 offset:32530; GFX1250-NEXT: s_wait_dscnt 0x0531; GFX1250-NEXT: s_set_pc_i64 s[30:31]532 %gep = getelementptr inbounds half, ptr addrspace(3) %ptr, i32 16533 %load = load atomic half, ptr addrspace(3) %gep monotonic, align 2534 %ret = bitcast half %load to i16535 ret i16 %ret536}537 538define i16 @atomic_load_monotonic_bf16(ptr addrspace(3) %ptr) {539; CI-LABEL: atomic_load_monotonic_bf16:540; CI: ; %bb.0:541; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)542; CI-NEXT: s_mov_b32 m0, -1543; CI-NEXT: ds_read_u16 v0, v0544; CI-NEXT: s_waitcnt lgkmcnt(0)545; CI-NEXT: s_setpc_b64 s[30:31]546;547; GFX9-LABEL: atomic_load_monotonic_bf16:548; GFX9: ; %bb.0:549; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)550; GFX9-NEXT: ds_read_u16 v0, v0551; GFX9-NEXT: s_waitcnt lgkmcnt(0)552; GFX9-NEXT: s_setpc_b64 s[30:31]553;554; GFX11-TRUE16-LABEL: atomic_load_monotonic_bf16:555; GFX11-TRUE16: ; %bb.0:556; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)557; GFX11-TRUE16-NEXT: ds_load_u16_d16 v0, v0558; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)559; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]560;561; GFX11-FAKE16-LABEL: atomic_load_monotonic_bf16:562; GFX11-FAKE16: ; %bb.0:563; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)564; GFX11-FAKE16-NEXT: ds_load_u16 v0, v0565; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)566; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]567;568; GFX1250-LABEL: atomic_load_monotonic_bf16:569; GFX1250: ; %bb.0:570; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0571; GFX1250-NEXT: s_wait_kmcnt 0x0572; GFX1250-NEXT: ds_load_u16 v0, v0573; GFX1250-NEXT: s_wait_dscnt 0x0574; GFX1250-NEXT: s_set_pc_i64 s[30:31]575 %load = load atomic bfloat, ptr addrspace(3) %ptr monotonic, align 2576 %ret = bitcast bfloat %load to i16577 ret i16 %ret578}579 580define i16 @atomic_load_monotonic_bf16_offset(ptr addrspace(3) %ptr) {581; CI-LABEL: atomic_load_monotonic_bf16_offset:582; CI: ; %bb.0:583; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)584; CI-NEXT: s_mov_b32 m0, -1585; CI-NEXT: ds_read_u16 v0, v0 offset:32586; CI-NEXT: s_waitcnt lgkmcnt(0)587; CI-NEXT: s_setpc_b64 s[30:31]588;589; GFX9-LABEL: atomic_load_monotonic_bf16_offset:590; GFX9: ; %bb.0:591; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)592; GFX9-NEXT: ds_read_u16 v0, v0 offset:32593; GFX9-NEXT: s_waitcnt lgkmcnt(0)594; GFX9-NEXT: s_setpc_b64 s[30:31]595;596; GFX11-TRUE16-LABEL: atomic_load_monotonic_bf16_offset:597; GFX11-TRUE16: ; %bb.0:598; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)599; GFX11-TRUE16-NEXT: ds_load_u16_d16 v0, v0 offset:32600; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)601; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]602;603; GFX11-FAKE16-LABEL: atomic_load_monotonic_bf16_offset:604; GFX11-FAKE16: ; %bb.0:605; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)606; GFX11-FAKE16-NEXT: ds_load_u16 v0, v0 offset:32607; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)608; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]609;610; GFX1250-LABEL: atomic_load_monotonic_bf16_offset:611; GFX1250: ; %bb.0:612; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0613; GFX1250-NEXT: s_wait_kmcnt 0x0614; GFX1250-NEXT: ds_load_u16 v0, v0 offset:32615; GFX1250-NEXT: s_wait_dscnt 0x0616; GFX1250-NEXT: s_set_pc_i64 s[30:31]617 %gep = getelementptr inbounds bfloat, ptr addrspace(3) %ptr, i32 16618 %load = load atomic bfloat, ptr addrspace(3) %gep monotonic, align 2619 %ret = bitcast bfloat %load to i16620 ret i16 %ret621}622;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:623; GCN: {{.*}}624; GFX1250-FAKE16: {{.*}}625; GFX1250-TRUE16: {{.*}}626