667 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri < %s | FileCheck -check-prefixes=GCN,CI %s3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s8 9define void @atomic_store_monotonic_i8(ptr addrspace(3) %ptr, i8 %val) {10; CI-LABEL: atomic_store_monotonic_i8:11; CI: ; %bb.0:12; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13; CI-NEXT: s_mov_b32 m0, -114; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v115; CI-NEXT: ds_write_b8 v0, v116; CI-NEXT: ds_write_b8 v0, v217; CI-NEXT: s_waitcnt lgkmcnt(0)18; CI-NEXT: s_setpc_b64 s[30:31]19;20; GFX9-LABEL: atomic_store_monotonic_i8:21; GFX9: ; %bb.0:22; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)23; GFX9-NEXT: v_add_u16_e32 v2, 2, v124; GFX9-NEXT: ds_write_b8 v0, v125; GFX9-NEXT: ds_write_b8 v0, v226; GFX9-NEXT: s_waitcnt lgkmcnt(0)27; GFX9-NEXT: s_setpc_b64 s[30:31]28;29; GFX11-TRUE16-LABEL: atomic_store_monotonic_i8:30; GFX11-TRUE16: ; %bb.0:31; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)32; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 233; GFX11-TRUE16-NEXT: ds_store_b8 v0, v134; GFX11-TRUE16-NEXT: ds_store_b8_d16_hi v0, v135; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)36; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]37;38; GFX11-FAKE16-LABEL: atomic_store_monotonic_i8:39; GFX11-FAKE16: ; %bb.0:40; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)41; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 242; GFX11-FAKE16-NEXT: ds_store_b8 v0, v143; GFX11-FAKE16-NEXT: ds_store_b8 v0, v244; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)45; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]46;47; GFX1250-TRUE16-LABEL: atomic_store_monotonic_i8:48; GFX1250-TRUE16: ; %bb.0:49; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x050; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x051; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 252; GFX1250-TRUE16-NEXT: ds_store_b8 v0, v153; GFX1250-TRUE16-NEXT: ds_store_b8_d16_hi v0, v154; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x055; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]56;57; GFX1250-FAKE16-LABEL: atomic_store_monotonic_i8:58; GFX1250-FAKE16: ; %bb.0:59; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x060; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x061; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 262; GFX1250-FAKE16-NEXT: ds_store_b8 v0, v163; GFX1250-FAKE16-NEXT: ds_store_b8 v0, v264; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x065; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]66 %val1 = add i8 %val, 267 store atomic i8 %val, ptr addrspace(3) %ptr monotonic, align 168 store atomic i8 %val1, ptr addrspace(3) %ptr monotonic, align 169 ret void70}71 72define void @atomic_store_monotonic_offset_i8(ptr addrspace(3) %ptr, i8 %val) {73; CI-LABEL: atomic_store_monotonic_offset_i8:74; CI: ; %bb.0:75; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)76; CI-NEXT: s_mov_b32 m0, -177; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v178; CI-NEXT: ds_write_b8 v0, v1 offset:879; CI-NEXT: ds_write_b8 v0, v2 offset:1680; CI-NEXT: s_waitcnt lgkmcnt(0)81; CI-NEXT: s_setpc_b64 s[30:31]82;83; GFX9-LABEL: atomic_store_monotonic_offset_i8:84; GFX9: ; %bb.0:85; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)86; GFX9-NEXT: v_add_u16_e32 v2, 2, v187; GFX9-NEXT: ds_write_b8 v0, v1 offset:888; GFX9-NEXT: ds_write_b8 v0, v2 offset:1689; GFX9-NEXT: s_waitcnt lgkmcnt(0)90; GFX9-NEXT: s_setpc_b64 s[30:31]91;92; GFX11-TRUE16-LABEL: atomic_store_monotonic_offset_i8:93; GFX11-TRUE16: ; %bb.0:94; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)95; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 296; GFX11-TRUE16-NEXT: ds_store_b8 v0, v1 offset:897; GFX11-TRUE16-NEXT: ds_store_b8_d16_hi v0, v1 offset:1698; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)99; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]100;101; GFX11-FAKE16-LABEL: atomic_store_monotonic_offset_i8:102; GFX11-FAKE16: ; %bb.0:103; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)104; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2105; GFX11-FAKE16-NEXT: ds_store_b8 v0, v1 offset:8106; GFX11-FAKE16-NEXT: ds_store_b8 v0, v2 offset:16107; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)108; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]109;110; GFX1250-TRUE16-LABEL: atomic_store_monotonic_offset_i8:111; GFX1250-TRUE16: ; %bb.0:112; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0113; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0114; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2115; GFX1250-TRUE16-NEXT: ds_store_b8 v0, v1 offset:8116; GFX1250-TRUE16-NEXT: ds_store_b8_d16_hi v0, v1 offset:16117; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x0118; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]119;120; GFX1250-FAKE16-LABEL: atomic_store_monotonic_offset_i8:121; GFX1250-FAKE16: ; %bb.0:122; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0123; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0124; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2125; GFX1250-FAKE16-NEXT: ds_store_b8 v0, v1 offset:8126; GFX1250-FAKE16-NEXT: ds_store_b8 v0, v2 offset:16127; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x0128; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]129 %val1 = add i8 %val, 2130 %gep_1 = getelementptr inbounds i8, ptr addrspace(3) %ptr, i8 8131 %gep_2 = getelementptr inbounds i8, ptr addrspace(3) %ptr, i8 16132 store atomic i8 %val, ptr addrspace(3) %gep_1 monotonic, align 1133 store atomic i8 %val1, ptr addrspace(3) %gep_2 monotonic, align 1134 ret void135}136 137define void @atomic_store_monotonic_i16(ptr addrspace(3) %ptr, i16 %val) {138; CI-LABEL: atomic_store_monotonic_i16:139; CI: ; %bb.0:140; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)141; CI-NEXT: s_mov_b32 m0, -1142; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1143; CI-NEXT: ds_write_b16 v0, v1144; CI-NEXT: ds_write_b16 v0, v2145; CI-NEXT: s_waitcnt lgkmcnt(0)146; CI-NEXT: s_setpc_b64 s[30:31]147;148; GFX9-LABEL: atomic_store_monotonic_i16:149; GFX9: ; %bb.0:150; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)151; GFX9-NEXT: v_add_u16_e32 v2, 2, v1152; GFX9-NEXT: ds_write_b16 v0, v1153; GFX9-NEXT: ds_write_b16 v0, v2154; GFX9-NEXT: s_waitcnt lgkmcnt(0)155; GFX9-NEXT: s_setpc_b64 s[30:31]156;157; GFX11-TRUE16-LABEL: atomic_store_monotonic_i16:158; GFX11-TRUE16: ; %bb.0:159; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)160; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2161; GFX11-TRUE16-NEXT: ds_store_b16 v0, v1162; GFX11-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1163; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)164; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]165;166; GFX11-FAKE16-LABEL: atomic_store_monotonic_i16:167; GFX11-FAKE16: ; %bb.0:168; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)169; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2170; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1171; GFX11-FAKE16-NEXT: ds_store_b16 v0, v2172; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)173; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]174;175; GFX1250-TRUE16-LABEL: atomic_store_monotonic_i16:176; GFX1250-TRUE16: ; %bb.0:177; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0178; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0179; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2180; GFX1250-TRUE16-NEXT: ds_store_b16 v0, v1181; GFX1250-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1182; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x0183; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]184;185; GFX1250-FAKE16-LABEL: atomic_store_monotonic_i16:186; GFX1250-FAKE16: ; %bb.0:187; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0188; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0189; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2190; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v1191; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v2192; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x0193; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]194 %val1 = add i16 %val, 2195 store atomic i16 %val, ptr addrspace(3) %ptr monotonic, align 2196 store atomic i16 %val1, ptr addrspace(3) %ptr monotonic, align 2197 ret void198}199 200define void @atomic_store_monotonic_offset_i16(ptr addrspace(3) %ptr, i16 %val) {201; CI-LABEL: atomic_store_monotonic_offset_i16:202; CI: ; %bb.0:203; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)204; CI-NEXT: s_mov_b32 m0, -1205; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1206; CI-NEXT: ds_write_b16 v0, v1 offset:32207; CI-NEXT: ds_write_b16 v0, v2 offset:32208; CI-NEXT: s_waitcnt lgkmcnt(0)209; CI-NEXT: s_setpc_b64 s[30:31]210;211; GFX9-LABEL: atomic_store_monotonic_offset_i16:212; GFX9: ; %bb.0:213; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)214; GFX9-NEXT: v_add_u16_e32 v2, 2, v1215; GFX9-NEXT: ds_write_b16 v0, v1 offset:32216; GFX9-NEXT: ds_write_b16 v0, v2 offset:32217; GFX9-NEXT: s_waitcnt lgkmcnt(0)218; GFX9-NEXT: s_setpc_b64 s[30:31]219;220; GFX11-TRUE16-LABEL: atomic_store_monotonic_offset_i16:221; GFX11-TRUE16: ; %bb.0:222; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)223; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2224; GFX11-TRUE16-NEXT: ds_store_b16 v0, v1 offset:32225; GFX11-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1 offset:32226; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)227; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]228;229; GFX11-FAKE16-LABEL: atomic_store_monotonic_offset_i16:230; GFX11-FAKE16: ; %bb.0:231; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)232; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2233; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1 offset:32234; GFX11-FAKE16-NEXT: ds_store_b16 v0, v2 offset:32235; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)236; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]237;238; GFX1250-TRUE16-LABEL: atomic_store_monotonic_offset_i16:239; GFX1250-TRUE16: ; %bb.0:240; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0241; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0242; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2243; GFX1250-TRUE16-NEXT: ds_store_b16 v0, v1 offset:32244; GFX1250-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1 offset:32245; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x0246; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]247;248; GFX1250-FAKE16-LABEL: atomic_store_monotonic_offset_i16:249; GFX1250-FAKE16: ; %bb.0:250; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0251; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0252; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2253; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v1 offset:32254; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v2 offset:32255; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x0256; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]257 %val1 = add i16 %val, 2258 %gep = getelementptr inbounds i16, ptr addrspace(3) %ptr, i16 16259 store atomic i16 %val, ptr addrspace(3) %gep monotonic, align 2260 store atomic i16 %val1, ptr addrspace(3) %gep monotonic, align 2261 ret void262}263 264define void @atomic_store_monotonic_i32(ptr addrspace(3) %ptr, i32 %val) {265; CI-LABEL: atomic_store_monotonic_i32:266; CI: ; %bb.0:267; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)268; CI-NEXT: s_mov_b32 m0, -1269; CI-NEXT: ds_write_b32 v0, v1270; CI-NEXT: s_waitcnt lgkmcnt(0)271; CI-NEXT: s_setpc_b64 s[30:31]272;273; GFX9-LABEL: atomic_store_monotonic_i32:274; GFX9: ; %bb.0:275; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)276; GFX9-NEXT: ds_write_b32 v0, v1277; GFX9-NEXT: s_waitcnt lgkmcnt(0)278; GFX9-NEXT: s_setpc_b64 s[30:31]279;280; GFX11-LABEL: atomic_store_monotonic_i32:281; GFX11: ; %bb.0:282; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)283; GFX11-NEXT: ds_store_b32 v0, v1284; GFX11-NEXT: s_waitcnt lgkmcnt(0)285; GFX11-NEXT: s_setpc_b64 s[30:31]286;287; GFX1250-LABEL: atomic_store_monotonic_i32:288; GFX1250: ; %bb.0:289; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0290; GFX1250-NEXT: s_wait_kmcnt 0x0291; GFX1250-NEXT: ds_store_b32 v0, v1292; GFX1250-NEXT: s_wait_dscnt 0x0293; GFX1250-NEXT: s_set_pc_i64 s[30:31]294 store atomic i32 %val, ptr addrspace(3) %ptr monotonic, align 4295 ret void296}297 298define void @atomic_store_monotonic_offset_i32(ptr addrspace(3) %ptr, i32 %val) {299; CI-LABEL: atomic_store_monotonic_offset_i32:300; CI: ; %bb.0:301; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)302; CI-NEXT: s_mov_b32 m0, -1303; CI-NEXT: ds_write_b32 v0, v1 offset:64304; CI-NEXT: s_waitcnt lgkmcnt(0)305; CI-NEXT: s_setpc_b64 s[30:31]306;307; GFX9-LABEL: atomic_store_monotonic_offset_i32:308; GFX9: ; %bb.0:309; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)310; GFX9-NEXT: ds_write_b32 v0, v1 offset:64311; GFX9-NEXT: s_waitcnt lgkmcnt(0)312; GFX9-NEXT: s_setpc_b64 s[30:31]313;314; GFX11-LABEL: atomic_store_monotonic_offset_i32:315; GFX11: ; %bb.0:316; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)317; GFX11-NEXT: ds_store_b32 v0, v1 offset:64318; GFX11-NEXT: s_waitcnt lgkmcnt(0)319; GFX11-NEXT: s_setpc_b64 s[30:31]320;321; GFX1250-LABEL: atomic_store_monotonic_offset_i32:322; GFX1250: ; %bb.0:323; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0324; GFX1250-NEXT: s_wait_kmcnt 0x0325; GFX1250-NEXT: ds_store_b32 v0, v1 offset:64326; GFX1250-NEXT: s_wait_dscnt 0x0327; GFX1250-NEXT: s_set_pc_i64 s[30:31]328 %gep = getelementptr inbounds i32, ptr addrspace(3) %ptr, i32 16329 store atomic i32 %val, ptr addrspace(3) %gep monotonic, align 4330 ret void331}332 333define void @atomic_store_monotonic_i64(ptr addrspace(3) %ptr, i64 %val) {334; CI-LABEL: atomic_store_monotonic_i64:335; CI: ; %bb.0:336; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)337; CI-NEXT: s_mov_b32 m0, -1338; CI-NEXT: ds_write_b64 v0, v[1:2]339; CI-NEXT: s_waitcnt lgkmcnt(0)340; CI-NEXT: s_setpc_b64 s[30:31]341;342; GFX9-LABEL: atomic_store_monotonic_i64:343; GFX9: ; %bb.0:344; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)345; GFX9-NEXT: ds_write_b64 v0, v[1:2]346; GFX9-NEXT: s_waitcnt lgkmcnt(0)347; GFX9-NEXT: s_setpc_b64 s[30:31]348;349; GFX11-LABEL: atomic_store_monotonic_i64:350; GFX11: ; %bb.0:351; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)352; GFX11-NEXT: ds_store_b64 v0, v[1:2]353; GFX11-NEXT: s_waitcnt lgkmcnt(0)354; GFX11-NEXT: s_setpc_b64 s[30:31]355;356; GFX1250-LABEL: atomic_store_monotonic_i64:357; GFX1250: ; %bb.0:358; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0359; GFX1250-NEXT: s_wait_kmcnt 0x0360; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1361; GFX1250-NEXT: ds_store_b64 v0, v[2:3]362; GFX1250-NEXT: s_wait_dscnt 0x0363; GFX1250-NEXT: s_set_pc_i64 s[30:31]364 store atomic i64 %val, ptr addrspace(3) %ptr monotonic, align 8365 ret void366}367 368define void @atomic_store_monotonic_offset_i64(ptr addrspace(3) %ptr, i64 %val) {369; CI-LABEL: atomic_store_monotonic_offset_i64:370; CI: ; %bb.0:371; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)372; CI-NEXT: s_mov_b32 m0, -1373; CI-NEXT: ds_write_b64 v0, v[1:2] offset:128374; CI-NEXT: s_waitcnt lgkmcnt(0)375; CI-NEXT: s_setpc_b64 s[30:31]376;377; GFX9-LABEL: atomic_store_monotonic_offset_i64:378; GFX9: ; %bb.0:379; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)380; GFX9-NEXT: ds_write_b64 v0, v[1:2] offset:128381; GFX9-NEXT: s_waitcnt lgkmcnt(0)382; GFX9-NEXT: s_setpc_b64 s[30:31]383;384; GFX11-LABEL: atomic_store_monotonic_offset_i64:385; GFX11: ; %bb.0:386; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)387; GFX11-NEXT: ds_store_b64 v0, v[1:2] offset:128388; GFX11-NEXT: s_waitcnt lgkmcnt(0)389; GFX11-NEXT: s_setpc_b64 s[30:31]390;391; GFX1250-LABEL: atomic_store_monotonic_offset_i64:392; GFX1250: ; %bb.0:393; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0394; GFX1250-NEXT: s_wait_kmcnt 0x0395; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1396; GFX1250-NEXT: ds_store_b64 v0, v[2:3] offset:128397; GFX1250-NEXT: s_wait_dscnt 0x0398; GFX1250-NEXT: s_set_pc_i64 s[30:31]399 %gep = getelementptr inbounds i64, ptr addrspace(3) %ptr, i64 16400 store atomic i64 %val, ptr addrspace(3) %gep monotonic, align 8401 ret void402}403 404define void @atomic_store_monotonic_f16(ptr addrspace(3) %ptr, i16 %arg.val) {405; CI-LABEL: atomic_store_monotonic_f16:406; CI: ; %bb.0:407; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)408; CI-NEXT: s_mov_b32 m0, -1409; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1410; CI-NEXT: ds_write_b16 v0, v1411; CI-NEXT: ds_write_b16 v0, v2412; CI-NEXT: s_waitcnt lgkmcnt(0)413; CI-NEXT: s_setpc_b64 s[30:31]414;415; GFX9-LABEL: atomic_store_monotonic_f16:416; GFX9: ; %bb.0:417; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)418; GFX9-NEXT: v_add_u16_e32 v2, 2, v1419; GFX9-NEXT: ds_write_b16 v0, v1420; GFX9-NEXT: ds_write_b16 v0, v2421; GFX9-NEXT: s_waitcnt lgkmcnt(0)422; GFX9-NEXT: s_setpc_b64 s[30:31]423;424; GFX11-TRUE16-LABEL: atomic_store_monotonic_f16:425; GFX11-TRUE16: ; %bb.0:426; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)427; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2428; GFX11-TRUE16-NEXT: ds_store_b16 v0, v1429; GFX11-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1430; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)431; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]432;433; GFX11-FAKE16-LABEL: atomic_store_monotonic_f16:434; GFX11-FAKE16: ; %bb.0:435; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)436; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2437; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1438; GFX11-FAKE16-NEXT: ds_store_b16 v0, v2439; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)440; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]441;442; GFX1250-TRUE16-LABEL: atomic_store_monotonic_f16:443; GFX1250-TRUE16: ; %bb.0:444; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0445; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0446; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2447; GFX1250-TRUE16-NEXT: ds_store_b16 v0, v1448; GFX1250-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1449; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x0450; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]451;452; GFX1250-FAKE16-LABEL: atomic_store_monotonic_f16:453; GFX1250-FAKE16: ; %bb.0:454; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0455; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0456; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2457; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v1458; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v2459; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x0460; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]461 %arg.val1 = add i16 %arg.val, 2462 %val = bitcast i16 %arg.val to half463 %val1 = bitcast i16 %arg.val1 to half464 store atomic half %val, ptr addrspace(3) %ptr monotonic, align 2465 store atomic half %val1, ptr addrspace(3) %ptr monotonic, align 2466 ret void467}468 469define void @atomic_store_monotonic_offset_f16(ptr addrspace(3) %ptr, i16 %arg.val) {470; CI-LABEL: atomic_store_monotonic_offset_f16:471; CI: ; %bb.0:472; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)473; CI-NEXT: s_mov_b32 m0, -1474; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1475; CI-NEXT: ds_write_b16 v0, v1 offset:32476; CI-NEXT: ds_write_b16 v0, v2 offset:32477; CI-NEXT: s_waitcnt lgkmcnt(0)478; CI-NEXT: s_setpc_b64 s[30:31]479;480; GFX9-LABEL: atomic_store_monotonic_offset_f16:481; GFX9: ; %bb.0:482; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)483; GFX9-NEXT: v_add_u16_e32 v2, 2, v1484; GFX9-NEXT: ds_write_b16 v0, v1 offset:32485; GFX9-NEXT: ds_write_b16 v0, v2 offset:32486; GFX9-NEXT: s_waitcnt lgkmcnt(0)487; GFX9-NEXT: s_setpc_b64 s[30:31]488;489; GFX11-TRUE16-LABEL: atomic_store_monotonic_offset_f16:490; GFX11-TRUE16: ; %bb.0:491; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)492; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2493; GFX11-TRUE16-NEXT: ds_store_b16 v0, v1 offset:32494; GFX11-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1 offset:32495; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)496; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]497;498; GFX11-FAKE16-LABEL: atomic_store_monotonic_offset_f16:499; GFX11-FAKE16: ; %bb.0:500; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)501; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2502; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1 offset:32503; GFX11-FAKE16-NEXT: ds_store_b16 v0, v2 offset:32504; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)505; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]506;507; GFX1250-TRUE16-LABEL: atomic_store_monotonic_offset_f16:508; GFX1250-TRUE16: ; %bb.0:509; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0510; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0511; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2512; GFX1250-TRUE16-NEXT: ds_store_b16 v0, v1 offset:32513; GFX1250-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1 offset:32514; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x0515; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]516;517; GFX1250-FAKE16-LABEL: atomic_store_monotonic_offset_f16:518; GFX1250-FAKE16: ; %bb.0:519; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0520; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0521; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2522; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v1 offset:32523; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v2 offset:32524; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x0525; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]526 %arg.val1 = add i16 %arg.val, 2527 %val1 = bitcast i16 %arg.val1 to half528 %val = bitcast i16 %arg.val to half529 %gep = getelementptr inbounds half, ptr addrspace(3) %ptr, i32 16530 store atomic half %val, ptr addrspace(3) %gep monotonic, align 2531 store atomic half %val1, ptr addrspace(3) %gep monotonic, align 2532 ret void533}534 535define void @atomic_store_monotonic_bf16(ptr addrspace(3) %ptr, i16 %arg.val) {536; CI-LABEL: atomic_store_monotonic_bf16:537; CI: ; %bb.0:538; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)539; CI-NEXT: s_mov_b32 m0, -1540; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1541; CI-NEXT: ds_write_b16 v0, v1542; CI-NEXT: ds_write_b16 v0, v2543; CI-NEXT: s_waitcnt lgkmcnt(0)544; CI-NEXT: s_setpc_b64 s[30:31]545;546; GFX9-LABEL: atomic_store_monotonic_bf16:547; GFX9: ; %bb.0:548; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)549; GFX9-NEXT: v_add_u16_e32 v2, 2, v1550; GFX9-NEXT: ds_write_b16 v0, v1551; GFX9-NEXT: ds_write_b16 v0, v2552; GFX9-NEXT: s_waitcnt lgkmcnt(0)553; GFX9-NEXT: s_setpc_b64 s[30:31]554;555; GFX11-TRUE16-LABEL: atomic_store_monotonic_bf16:556; GFX11-TRUE16: ; %bb.0:557; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)558; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2559; GFX11-TRUE16-NEXT: ds_store_b16 v0, v1560; GFX11-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1561; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)562; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]563;564; GFX11-FAKE16-LABEL: atomic_store_monotonic_bf16:565; GFX11-FAKE16: ; %bb.0:566; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)567; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2568; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1569; GFX11-FAKE16-NEXT: ds_store_b16 v0, v2570; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)571; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]572;573; GFX1250-TRUE16-LABEL: atomic_store_monotonic_bf16:574; GFX1250-TRUE16: ; %bb.0:575; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0576; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0577; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2578; GFX1250-TRUE16-NEXT: ds_store_b16 v0, v1579; GFX1250-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1580; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x0581; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]582;583; GFX1250-FAKE16-LABEL: atomic_store_monotonic_bf16:584; GFX1250-FAKE16: ; %bb.0:585; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0586; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0587; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2588; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v1589; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v2590; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x0591; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]592 %arg.val1 = add i16 %arg.val, 2593 %val1 = bitcast i16 %arg.val1 to bfloat594 %val = bitcast i16 %arg.val to bfloat595 store atomic bfloat %val, ptr addrspace(3) %ptr monotonic, align 2596 store atomic bfloat %val1, ptr addrspace(3) %ptr monotonic, align 2597 ret void598}599 600define void @atomic_store_monotonic_offset_bf16(ptr addrspace(3) %ptr, i16 %arg.val) {601; CI-LABEL: atomic_store_monotonic_offset_bf16:602; CI: ; %bb.0:603; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)604; CI-NEXT: s_mov_b32 m0, -1605; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1606; CI-NEXT: ds_write_b16 v0, v1 offset:32607; CI-NEXT: ds_write_b16 v0, v2 offset:32608; CI-NEXT: s_waitcnt lgkmcnt(0)609; CI-NEXT: s_setpc_b64 s[30:31]610;611; GFX9-LABEL: atomic_store_monotonic_offset_bf16:612; GFX9: ; %bb.0:613; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)614; GFX9-NEXT: v_add_u16_e32 v2, 2, v1615; GFX9-NEXT: ds_write_b16 v0, v1 offset:32616; GFX9-NEXT: ds_write_b16 v0, v2 offset:32617; GFX9-NEXT: s_waitcnt lgkmcnt(0)618; GFX9-NEXT: s_setpc_b64 s[30:31]619;620; GFX11-TRUE16-LABEL: atomic_store_monotonic_offset_bf16:621; GFX11-TRUE16: ; %bb.0:622; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)623; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2624; GFX11-TRUE16-NEXT: ds_store_b16 v0, v1 offset:32625; GFX11-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1 offset:32626; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)627; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]628;629; GFX11-FAKE16-LABEL: atomic_store_monotonic_offset_bf16:630; GFX11-FAKE16: ; %bb.0:631; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)632; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2633; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1 offset:32634; GFX11-FAKE16-NEXT: ds_store_b16 v0, v2 offset:32635; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)636; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]637;638; GFX1250-TRUE16-LABEL: atomic_store_monotonic_offset_bf16:639; GFX1250-TRUE16: ; %bb.0:640; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0641; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0642; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.l, 2643; GFX1250-TRUE16-NEXT: ds_store_b16 v0, v1 offset:32644; GFX1250-TRUE16-NEXT: ds_store_b16_d16_hi v0, v1 offset:32645; GFX1250-TRUE16-NEXT: s_wait_dscnt 0x0646; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]647;648; GFX1250-FAKE16-LABEL: atomic_store_monotonic_offset_bf16:649; GFX1250-FAKE16: ; %bb.0:650; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0651; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0652; GFX1250-FAKE16-NEXT: v_add_nc_u16 v2, v1, 2653; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v1 offset:32654; GFX1250-FAKE16-NEXT: ds_store_b16 v0, v2 offset:32655; GFX1250-FAKE16-NEXT: s_wait_dscnt 0x0656; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]657 %arg.val1 = add i16 %arg.val, 2658 %val1 = bitcast i16 %arg.val1 to bfloat659 %val = bitcast i16 %arg.val to bfloat660 %gep = getelementptr inbounds bfloat, ptr addrspace(3) %ptr, i32 16661 store atomic bfloat %val, ptr addrspace(3) %gep monotonic, align 2662 store atomic bfloat %val1, ptr addrspace(3) %gep monotonic, align 2663 ret void664}665;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:666; GCN: {{.*}}667