5057 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx82 | FileCheck %s3; RUN: %if ptxas-sm_70 && ptxas-isa-8.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx82 | %ptxas-verify -arch=sm_70 %}4 5; TODO: fix "atomic load volatile acquire": generates "ld.acquire.sys;"6; but should generate "ld.mmio.relaxed.sys; fence.acq_rel.sys;"7; TODO: fix "atomic store volatile release": generates "st.release.sys;"8; but should generate "fence.acq_rel.sys; st.mmio.relaxed.sys;"9 10; TODO: fix "atomic load volatile seq_cst": generates "fence.sc.sys; ld.acquire.sys;"11; but should generate "fence.sc.sys; ld.relaxed.mmio.sys; fence.acq_rel.sys;"12; TODO: fix "atomic store volatile seq_cst": generates "fence.sc.sys; st.release.sys;"13; but should generate "fence.sc.sys; st.relaxed.mmio.sys;"14 15; TODO: add i1, <8 x i8>, and <6 x i8> vector tests.16 17; TODO: add test for vectors that exceed 128-bit length18; Per https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#vectors19; vectors cannot exceed 128-bit in length, i.e., .v4.u64 is not allowed.20 21; TODO: generate PTX that preserves Concurrent Forward Progress22; for atomic operations to local statespace23; by generating atomic or volatile operations.24 25; TODO: design exposure for atomic operations on vector types.26 27; TODO: implement and test thread scope.28 29; TODO: add weak,atomic,volatile,atomic volatile tests30; for .const and .param statespaces.31 32; TODO: optimize .sys.shared into .cta.shared or .cluster.shared .33 34;; generic statespace35 36; CHECK-LABEL: generic_unordered_gpu37define void @generic_unordered_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {38; CHECK-LABEL: generic_unordered_gpu(39; CHECK: {40; CHECK-NEXT: .reg .b16 %rs<5>;41; CHECK-NEXT: .reg .b32 %r<5>;42; CHECK-NEXT: .reg .b64 %rd<10>;43; CHECK-EMPTY:44; CHECK-NEXT: // %bb.0:45; CHECK-NEXT: ld.param.b64 %rd1, [generic_unordered_gpu_param_0];46; CHECK-NEXT: ld.relaxed.gpu.b8 %rs1, [%rd1];47; CHECK-NEXT: ld.param.b64 %rd2, [generic_unordered_gpu_param_1];48; CHECK-NEXT: add.s16 %rs2, %rs1, 1;49; CHECK-NEXT: ld.param.b64 %rd3, [generic_unordered_gpu_param_2];50; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs2;51; CHECK-NEXT: ld.param.b64 %rd4, [generic_unordered_gpu_param_3];52; CHECK-NEXT: ld.relaxed.gpu.b16 %rs3, [%rd2];53; CHECK-NEXT: ld.param.b64 %rd5, [generic_unordered_gpu_param_4];54; CHECK-NEXT: add.s16 %rs4, %rs3, 1;55; CHECK-NEXT: st.relaxed.gpu.b16 [%rd2], %rs4;56; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd3];57; CHECK-NEXT: add.s32 %r2, %r1, 1;58; CHECK-NEXT: st.relaxed.gpu.b32 [%rd3], %r2;59; CHECK-NEXT: ld.relaxed.gpu.b64 %rd6, [%rd4];60; CHECK-NEXT: add.s64 %rd7, %rd6, 1;61; CHECK-NEXT: st.relaxed.gpu.b64 [%rd4], %rd7;62; CHECK-NEXT: ld.relaxed.gpu.b32 %r3, [%rd5];63; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;64; CHECK-NEXT: st.relaxed.gpu.b32 [%rd5], %r4;65; CHECK-NEXT: ld.relaxed.gpu.b64 %rd8, [%rd5];66; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;67; CHECK-NEXT: st.relaxed.gpu.b64 [%rd5], %rd9;68; CHECK-NEXT: ret;69 %a.load = load atomic i8, ptr %a syncscope("device") unordered, align 170 %a.add = add i8 %a.load, 171 store atomic i8 %a.add, ptr %a syncscope("device") unordered, align 172 73 %b.load = load atomic i16, ptr %b syncscope("device") unordered, align 274 %b.add = add i16 %b.load, 175 store atomic i16 %b.add, ptr %b syncscope("device") unordered, align 276 77 %c.load = load atomic i32, ptr %c syncscope("device") unordered, align 478 %c.add = add i32 %c.load, 179 store atomic i32 %c.add, ptr %c syncscope("device") unordered, align 480 81 %d.load = load atomic i64, ptr %d syncscope("device") unordered, align 882 %d.add = add i64 %d.load, 183 store atomic i64 %d.add, ptr %d syncscope("device") unordered, align 884 85 %e.load = load atomic float, ptr %e syncscope("device") unordered, align 486 %e.add = fadd float %e.load, 1.87 store atomic float %e.add, ptr %e syncscope("device") unordered, align 488 89 %f.load = load atomic double, ptr %e syncscope("device") unordered, align 890 %f.add = fadd double %f.load, 1.91 store atomic double %f.add, ptr %e syncscope("device") unordered, align 892 93 ret void94}95 96; CHECK-LABEL: generic_unordered_volatile_gpu97define void @generic_unordered_volatile_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {98; CHECK-LABEL: generic_unordered_volatile_gpu(99; CHECK: {100; CHECK-NEXT: .reg .b16 %rs<5>;101; CHECK-NEXT: .reg .b32 %r<5>;102; CHECK-NEXT: .reg .b64 %rd<10>;103; CHECK-EMPTY:104; CHECK-NEXT: // %bb.0:105; CHECK-NEXT: ld.param.b64 %rd1, [generic_unordered_volatile_gpu_param_0];106; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];107; CHECK-NEXT: ld.param.b64 %rd2, [generic_unordered_volatile_gpu_param_1];108; CHECK-NEXT: add.s16 %rs2, %rs1, 1;109; CHECK-NEXT: ld.param.b64 %rd3, [generic_unordered_volatile_gpu_param_2];110; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;111; CHECK-NEXT: ld.param.b64 %rd4, [generic_unordered_volatile_gpu_param_3];112; CHECK-NEXT: ld.volatile.b16 %rs3, [%rd2];113; CHECK-NEXT: ld.param.b64 %rd5, [generic_unordered_volatile_gpu_param_4];114; CHECK-NEXT: add.s16 %rs4, %rs3, 1;115; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;116; CHECK-NEXT: ld.volatile.b32 %r1, [%rd3];117; CHECK-NEXT: add.s32 %r2, %r1, 1;118; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;119; CHECK-NEXT: ld.volatile.b64 %rd6, [%rd4];120; CHECK-NEXT: add.s64 %rd7, %rd6, 1;121; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;122; CHECK-NEXT: ld.volatile.b32 %r3, [%rd5];123; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;124; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;125; CHECK-NEXT: ld.volatile.b64 %rd8, [%rd5];126; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;127; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;128; CHECK-NEXT: ret;129 %a.load = load atomic volatile i8, ptr %a syncscope("device") unordered, align 1130 %a.add = add i8 %a.load, 1131 store atomic volatile i8 %a.add, ptr %a syncscope("device") unordered, align 1132 133 %b.load = load atomic volatile i16, ptr %b syncscope("device") unordered, align 2134 %b.add = add i16 %b.load, 1135 store atomic volatile i16 %b.add, ptr %b syncscope("device") unordered, align 2136 137 %c.load = load atomic volatile i32, ptr %c syncscope("device") unordered, align 4138 %c.add = add i32 %c.load, 1139 store atomic volatile i32 %c.add, ptr %c syncscope("device") unordered, align 4140 141 %d.load = load atomic volatile i64, ptr %d syncscope("device") unordered, align 8142 %d.add = add i64 %d.load, 1143 store atomic volatile i64 %d.add, ptr %d syncscope("device") unordered, align 8144 145 %e.load = load atomic volatile float, ptr %e syncscope("device") unordered, align 4146 %e.add = fadd float %e.load, 1.147 store atomic volatile float %e.add, ptr %e syncscope("device") unordered, align 4148 149 %f.load = load atomic volatile double, ptr %e syncscope("device") unordered, align 8150 %f.add = fadd double %f.load, 1.151 store atomic volatile double %f.add, ptr %e syncscope("device") unordered, align 8152 153 ret void154}155 156; CHECK-LABEL: generic_unordered_cta157define void @generic_unordered_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {158; CHECK-LABEL: generic_unordered_cta(159; CHECK: {160; CHECK-NEXT: .reg .b16 %rs<5>;161; CHECK-NEXT: .reg .b32 %r<5>;162; CHECK-NEXT: .reg .b64 %rd<10>;163; CHECK-EMPTY:164; CHECK-NEXT: // %bb.0:165; CHECK-NEXT: ld.param.b64 %rd1, [generic_unordered_cta_param_0];166; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];167; CHECK-NEXT: ld.param.b64 %rd2, [generic_unordered_cta_param_1];168; CHECK-NEXT: add.s16 %rs2, %rs1, 1;169; CHECK-NEXT: ld.param.b64 %rd3, [generic_unordered_cta_param_2];170; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs2;171; CHECK-NEXT: ld.param.b64 %rd4, [generic_unordered_cta_param_3];172; CHECK-NEXT: ld.relaxed.cta.b16 %rs3, [%rd2];173; CHECK-NEXT: ld.param.b64 %rd5, [generic_unordered_cta_param_4];174; CHECK-NEXT: add.s16 %rs4, %rs3, 1;175; CHECK-NEXT: st.relaxed.cta.b16 [%rd2], %rs4;176; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd3];177; CHECK-NEXT: add.s32 %r2, %r1, 1;178; CHECK-NEXT: st.relaxed.cta.b32 [%rd3], %r2;179; CHECK-NEXT: ld.relaxed.cta.b64 %rd6, [%rd4];180; CHECK-NEXT: add.s64 %rd7, %rd6, 1;181; CHECK-NEXT: st.relaxed.cta.b64 [%rd4], %rd7;182; CHECK-NEXT: ld.relaxed.cta.b32 %r3, [%rd5];183; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;184; CHECK-NEXT: st.relaxed.cta.b32 [%rd5], %r4;185; CHECK-NEXT: ld.relaxed.cta.b64 %rd8, [%rd5];186; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;187; CHECK-NEXT: st.relaxed.cta.b64 [%rd5], %rd9;188; CHECK-NEXT: ret;189 %a.load = load atomic i8, ptr %a syncscope("block") unordered, align 1190 %a.add = add i8 %a.load, 1191 store atomic i8 %a.add, ptr %a syncscope("block") unordered, align 1192 193 %b.load = load atomic i16, ptr %b syncscope("block") unordered, align 2194 %b.add = add i16 %b.load, 1195 store atomic i16 %b.add, ptr %b syncscope("block") unordered, align 2196 197 %c.load = load atomic i32, ptr %c syncscope("block") unordered, align 4198 %c.add = add i32 %c.load, 1199 store atomic i32 %c.add, ptr %c syncscope("block") unordered, align 4200 201 %d.load = load atomic i64, ptr %d syncscope("block") unordered, align 8202 %d.add = add i64 %d.load, 1203 store atomic i64 %d.add, ptr %d syncscope("block") unordered, align 8204 205 %e.load = load atomic float, ptr %e syncscope("block") unordered, align 4206 %e.add = fadd float %e.load, 1.207 store atomic float %e.add, ptr %e syncscope("block") unordered, align 4208 209 %f.load = load atomic double, ptr %e syncscope("block") unordered, align 8210 %f.add = fadd double %f.load, 1.211 store atomic double %f.add, ptr %e syncscope("block") unordered, align 8212 213 ret void214}215 216; CHECK-LABEL: generic_unordered_volatile_cta217define void @generic_unordered_volatile_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {218; CHECK-LABEL: generic_unordered_volatile_cta(219; CHECK: {220; CHECK-NEXT: .reg .b16 %rs<5>;221; CHECK-NEXT: .reg .b32 %r<5>;222; CHECK-NEXT: .reg .b64 %rd<10>;223; CHECK-EMPTY:224; CHECK-NEXT: // %bb.0:225; CHECK-NEXT: ld.param.b64 %rd1, [generic_unordered_volatile_cta_param_0];226; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];227; CHECK-NEXT: ld.param.b64 %rd2, [generic_unordered_volatile_cta_param_1];228; CHECK-NEXT: add.s16 %rs2, %rs1, 1;229; CHECK-NEXT: ld.param.b64 %rd3, [generic_unordered_volatile_cta_param_2];230; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;231; CHECK-NEXT: ld.param.b64 %rd4, [generic_unordered_volatile_cta_param_3];232; CHECK-NEXT: ld.volatile.b16 %rs3, [%rd2];233; CHECK-NEXT: ld.param.b64 %rd5, [generic_unordered_volatile_cta_param_4];234; CHECK-NEXT: add.s16 %rs4, %rs3, 1;235; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;236; CHECK-NEXT: ld.volatile.b32 %r1, [%rd3];237; CHECK-NEXT: add.s32 %r2, %r1, 1;238; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;239; CHECK-NEXT: ld.volatile.b64 %rd6, [%rd4];240; CHECK-NEXT: add.s64 %rd7, %rd6, 1;241; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;242; CHECK-NEXT: ld.volatile.b32 %r3, [%rd5];243; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;244; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;245; CHECK-NEXT: ld.volatile.b64 %rd8, [%rd5];246; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;247; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;248; CHECK-NEXT: ret;249 %a.load = load atomic volatile i8, ptr %a syncscope("block") unordered, align 1250 %a.add = add i8 %a.load, 1251 store atomic volatile i8 %a.add, ptr %a syncscope("block") unordered, align 1252 253 %b.load = load atomic volatile i16, ptr %b syncscope("block") unordered, align 2254 %b.add = add i16 %b.load, 1255 store atomic volatile i16 %b.add, ptr %b syncscope("block") unordered, align 2256 257 %c.load = load atomic volatile i32, ptr %c syncscope("block") unordered, align 4258 %c.add = add i32 %c.load, 1259 store atomic volatile i32 %c.add, ptr %c syncscope("block") unordered, align 4260 261 %d.load = load atomic volatile i64, ptr %d syncscope("block") unordered, align 8262 %d.add = add i64 %d.load, 1263 store atomic volatile i64 %d.add, ptr %d syncscope("block") unordered, align 8264 265 %e.load = load atomic volatile float, ptr %e syncscope("block") unordered, align 4266 %e.add = fadd float %e.load, 1.267 store atomic volatile float %e.add, ptr %e syncscope("block") unordered, align 4268 269 %f.load = load atomic volatile double, ptr %e syncscope("block") unordered, align 8270 %f.add = fadd double %f.load, 1.271 store atomic volatile double %f.add, ptr %e syncscope("block") unordered, align 8272 273 ret void274}275 276; CHECK-LABEL: generic_monotonic_gpu277define void @generic_monotonic_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {278; CHECK-LABEL: generic_monotonic_gpu(279; CHECK: {280; CHECK-NEXT: .reg .b16 %rs<5>;281; CHECK-NEXT: .reg .b32 %r<5>;282; CHECK-NEXT: .reg .b64 %rd<10>;283; CHECK-EMPTY:284; CHECK-NEXT: // %bb.0:285; CHECK-NEXT: ld.param.b64 %rd1, [generic_monotonic_gpu_param_0];286; CHECK-NEXT: ld.relaxed.gpu.b8 %rs1, [%rd1];287; CHECK-NEXT: ld.param.b64 %rd2, [generic_monotonic_gpu_param_1];288; CHECK-NEXT: add.s16 %rs2, %rs1, 1;289; CHECK-NEXT: ld.param.b64 %rd3, [generic_monotonic_gpu_param_2];290; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs2;291; CHECK-NEXT: ld.param.b64 %rd4, [generic_monotonic_gpu_param_3];292; CHECK-NEXT: ld.relaxed.gpu.b16 %rs3, [%rd2];293; CHECK-NEXT: ld.param.b64 %rd5, [generic_monotonic_gpu_param_4];294; CHECK-NEXT: add.s16 %rs4, %rs3, 1;295; CHECK-NEXT: st.relaxed.gpu.b16 [%rd2], %rs4;296; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd3];297; CHECK-NEXT: add.s32 %r2, %r1, 1;298; CHECK-NEXT: st.relaxed.gpu.b32 [%rd3], %r2;299; CHECK-NEXT: ld.relaxed.gpu.b64 %rd6, [%rd4];300; CHECK-NEXT: add.s64 %rd7, %rd6, 1;301; CHECK-NEXT: st.relaxed.gpu.b64 [%rd4], %rd7;302; CHECK-NEXT: ld.relaxed.gpu.b32 %r3, [%rd5];303; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;304; CHECK-NEXT: st.relaxed.gpu.b32 [%rd5], %r4;305; CHECK-NEXT: ld.relaxed.gpu.b64 %rd8, [%rd5];306; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;307; CHECK-NEXT: st.relaxed.gpu.b64 [%rd5], %rd9;308; CHECK-NEXT: ret;309 %a.load = load atomic i8, ptr %a syncscope("device") monotonic, align 1310 %a.add = add i8 %a.load, 1311 store atomic i8 %a.add, ptr %a syncscope("device") monotonic, align 1312 313 %b.load = load atomic i16, ptr %b syncscope("device") monotonic, align 2314 %b.add = add i16 %b.load, 1315 store atomic i16 %b.add, ptr %b syncscope("device") monotonic, align 2316 317 %c.load = load atomic i32, ptr %c syncscope("device") monotonic, align 4318 %c.add = add i32 %c.load, 1319 store atomic i32 %c.add, ptr %c syncscope("device") monotonic, align 4320 321 %d.load = load atomic i64, ptr %d syncscope("device") monotonic, align 8322 %d.add = add i64 %d.load, 1323 store atomic i64 %d.add, ptr %d syncscope("device") monotonic, align 8324 325 %e.load = load atomic float, ptr %e syncscope("device") monotonic, align 4326 %e.add = fadd float %e.load, 1.327 store atomic float %e.add, ptr %e syncscope("device") monotonic, align 4328 329 %f.load = load atomic double, ptr %e syncscope("device") monotonic, align 8330 %f.add = fadd double %f.load, 1.331 store atomic double %f.add, ptr %e syncscope("device") monotonic, align 8332 333 ret void334}335 336; CHECK-LABEL: generic_monotonic_volatile_gpu337define void @generic_monotonic_volatile_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {338; CHECK-LABEL: generic_monotonic_volatile_gpu(339; CHECK: {340; CHECK-NEXT: .reg .b16 %rs<5>;341; CHECK-NEXT: .reg .b32 %r<5>;342; CHECK-NEXT: .reg .b64 %rd<10>;343; CHECK-EMPTY:344; CHECK-NEXT: // %bb.0:345; CHECK-NEXT: ld.param.b64 %rd1, [generic_monotonic_volatile_gpu_param_0];346; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];347; CHECK-NEXT: ld.param.b64 %rd2, [generic_monotonic_volatile_gpu_param_1];348; CHECK-NEXT: add.s16 %rs2, %rs1, 1;349; CHECK-NEXT: ld.param.b64 %rd3, [generic_monotonic_volatile_gpu_param_2];350; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;351; CHECK-NEXT: ld.param.b64 %rd4, [generic_monotonic_volatile_gpu_param_3];352; CHECK-NEXT: ld.volatile.b16 %rs3, [%rd2];353; CHECK-NEXT: ld.param.b64 %rd5, [generic_monotonic_volatile_gpu_param_4];354; CHECK-NEXT: add.s16 %rs4, %rs3, 1;355; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;356; CHECK-NEXT: ld.volatile.b32 %r1, [%rd3];357; CHECK-NEXT: add.s32 %r2, %r1, 1;358; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;359; CHECK-NEXT: ld.volatile.b64 %rd6, [%rd4];360; CHECK-NEXT: add.s64 %rd7, %rd6, 1;361; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;362; CHECK-NEXT: ld.volatile.b32 %r3, [%rd5];363; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;364; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;365; CHECK-NEXT: ld.volatile.b64 %rd8, [%rd5];366; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;367; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;368; CHECK-NEXT: ret;369 %a.load = load atomic volatile i8, ptr %a syncscope("device") monotonic, align 1370 %a.add = add i8 %a.load, 1371 store atomic volatile i8 %a.add, ptr %a syncscope("device") monotonic, align 1372 373 %b.load = load atomic volatile i16, ptr %b syncscope("device") monotonic, align 2374 %b.add = add i16 %b.load, 1375 store atomic volatile i16 %b.add, ptr %b syncscope("device") monotonic, align 2376 377 %c.load = load atomic volatile i32, ptr %c syncscope("device") monotonic, align 4378 %c.add = add i32 %c.load, 1379 store atomic volatile i32 %c.add, ptr %c syncscope("device") monotonic, align 4380 381 %d.load = load atomic volatile i64, ptr %d syncscope("device") monotonic, align 8382 %d.add = add i64 %d.load, 1383 store atomic volatile i64 %d.add, ptr %d syncscope("device") monotonic, align 8384 385 %e.load = load atomic volatile float, ptr %e syncscope("device") monotonic, align 4386 %e.add = fadd float %e.load, 1.387 store atomic volatile float %e.add, ptr %e syncscope("device") monotonic, align 4388 389 %f.load = load atomic volatile double, ptr %e syncscope("device") monotonic, align 8390 %f.add = fadd double %f.load, 1.391 store atomic volatile double %f.add, ptr %e syncscope("device") monotonic, align 8392 393 ret void394}395 396; CHECK-LABEL: generic_monotonic_cta397define void @generic_monotonic_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {398; CHECK-LABEL: generic_monotonic_cta(399; CHECK: {400; CHECK-NEXT: .reg .b16 %rs<5>;401; CHECK-NEXT: .reg .b32 %r<5>;402; CHECK-NEXT: .reg .b64 %rd<10>;403; CHECK-EMPTY:404; CHECK-NEXT: // %bb.0:405; CHECK-NEXT: ld.param.b64 %rd1, [generic_monotonic_cta_param_0];406; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];407; CHECK-NEXT: ld.param.b64 %rd2, [generic_monotonic_cta_param_1];408; CHECK-NEXT: add.s16 %rs2, %rs1, 1;409; CHECK-NEXT: ld.param.b64 %rd3, [generic_monotonic_cta_param_2];410; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs2;411; CHECK-NEXT: ld.param.b64 %rd4, [generic_monotonic_cta_param_3];412; CHECK-NEXT: ld.relaxed.cta.b16 %rs3, [%rd2];413; CHECK-NEXT: ld.param.b64 %rd5, [generic_monotonic_cta_param_4];414; CHECK-NEXT: add.s16 %rs4, %rs3, 1;415; CHECK-NEXT: st.relaxed.cta.b16 [%rd2], %rs4;416; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd3];417; CHECK-NEXT: add.s32 %r2, %r1, 1;418; CHECK-NEXT: st.relaxed.cta.b32 [%rd3], %r2;419; CHECK-NEXT: ld.relaxed.cta.b64 %rd6, [%rd4];420; CHECK-NEXT: add.s64 %rd7, %rd6, 1;421; CHECK-NEXT: st.relaxed.cta.b64 [%rd4], %rd7;422; CHECK-NEXT: ld.relaxed.cta.b32 %r3, [%rd5];423; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;424; CHECK-NEXT: st.relaxed.cta.b32 [%rd5], %r4;425; CHECK-NEXT: ld.relaxed.cta.b64 %rd8, [%rd5];426; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;427; CHECK-NEXT: st.relaxed.cta.b64 [%rd5], %rd9;428; CHECK-NEXT: ret;429 %a.load = load atomic i8, ptr %a syncscope("block") monotonic, align 1430 %a.add = add i8 %a.load, 1431 store atomic i8 %a.add, ptr %a syncscope("block") monotonic, align 1432 433 %b.load = load atomic i16, ptr %b syncscope("block") monotonic, align 2434 %b.add = add i16 %b.load, 1435 store atomic i16 %b.add, ptr %b syncscope("block") monotonic, align 2436 437 %c.load = load atomic i32, ptr %c syncscope("block") monotonic, align 4438 %c.add = add i32 %c.load, 1439 store atomic i32 %c.add, ptr %c syncscope("block") monotonic, align 4440 441 %d.load = load atomic i64, ptr %d syncscope("block") monotonic, align 8442 %d.add = add i64 %d.load, 1443 store atomic i64 %d.add, ptr %d syncscope("block") monotonic, align 8444 445 %e.load = load atomic float, ptr %e syncscope("block") monotonic, align 4446 %e.add = fadd float %e.load, 1.447 store atomic float %e.add, ptr %e syncscope("block") monotonic, align 4448 449 %f.load = load atomic double, ptr %e syncscope("block") monotonic, align 8450 %f.add = fadd double %f.load, 1.451 store atomic double %f.add, ptr %e syncscope("block") monotonic, align 8452 453 ret void454}455 456; CHECK-LABEL: generic_monotonic_volatile_cta457define void @generic_monotonic_volatile_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {458; CHECK-LABEL: generic_monotonic_volatile_cta(459; CHECK: {460; CHECK-NEXT: .reg .b16 %rs<5>;461; CHECK-NEXT: .reg .b32 %r<5>;462; CHECK-NEXT: .reg .b64 %rd<10>;463; CHECK-EMPTY:464; CHECK-NEXT: // %bb.0:465; CHECK-NEXT: ld.param.b64 %rd1, [generic_monotonic_volatile_cta_param_0];466; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];467; CHECK-NEXT: ld.param.b64 %rd2, [generic_monotonic_volatile_cta_param_1];468; CHECK-NEXT: add.s16 %rs2, %rs1, 1;469; CHECK-NEXT: ld.param.b64 %rd3, [generic_monotonic_volatile_cta_param_2];470; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;471; CHECK-NEXT: ld.param.b64 %rd4, [generic_monotonic_volatile_cta_param_3];472; CHECK-NEXT: ld.volatile.b16 %rs3, [%rd2];473; CHECK-NEXT: ld.param.b64 %rd5, [generic_monotonic_volatile_cta_param_4];474; CHECK-NEXT: add.s16 %rs4, %rs3, 1;475; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;476; CHECK-NEXT: ld.volatile.b32 %r1, [%rd3];477; CHECK-NEXT: add.s32 %r2, %r1, 1;478; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;479; CHECK-NEXT: ld.volatile.b64 %rd6, [%rd4];480; CHECK-NEXT: add.s64 %rd7, %rd6, 1;481; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;482; CHECK-NEXT: ld.volatile.b32 %r3, [%rd5];483; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;484; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;485; CHECK-NEXT: ld.volatile.b64 %rd8, [%rd5];486; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;487; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;488; CHECK-NEXT: ret;489 %a.load = load atomic volatile i8, ptr %a syncscope("block") monotonic, align 1490 %a.add = add i8 %a.load, 1491 store atomic volatile i8 %a.add, ptr %a syncscope("block") monotonic, align 1492 493 %b.load = load atomic volatile i16, ptr %b syncscope("block") monotonic, align 2494 %b.add = add i16 %b.load, 1495 store atomic volatile i16 %b.add, ptr %b syncscope("block") monotonic, align 2496 497 %c.load = load atomic volatile i32, ptr %c syncscope("block") monotonic, align 4498 %c.add = add i32 %c.load, 1499 store atomic volatile i32 %c.add, ptr %c syncscope("block") monotonic, align 4500 501 %d.load = load atomic volatile i64, ptr %d syncscope("block") monotonic, align 8502 %d.add = add i64 %d.load, 1503 store atomic volatile i64 %d.add, ptr %d syncscope("block") monotonic, align 8504 505 %e.load = load atomic volatile float, ptr %e syncscope("block") monotonic, align 4506 %e.add = fadd float %e.load, 1.507 store atomic volatile float %e.add, ptr %e syncscope("block") monotonic, align 4508 509 %f.load = load atomic volatile double, ptr %e syncscope("block") monotonic, align 8510 %f.add = fadd double %f.load, 1.511 store atomic volatile double %f.add, ptr %e syncscope("block") monotonic, align 8512 513 ret void514}515 516; CHECK-LABEL: generic_acq_rel_sys517define void @generic_acq_rel_sys(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {518; CHECK-LABEL: generic_acq_rel_sys(519; CHECK: {520; CHECK-NEXT: .reg .b16 %rs<5>;521; CHECK-NEXT: .reg .b32 %r<5>;522; CHECK-NEXT: .reg .b64 %rd<10>;523; CHECK-EMPTY:524; CHECK-NEXT: // %bb.0:525; CHECK-NEXT: ld.param.b64 %rd1, [generic_acq_rel_sys_param_0];526; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];527; CHECK-NEXT: ld.param.b64 %rd2, [generic_acq_rel_sys_param_1];528; CHECK-NEXT: add.s16 %rs2, %rs1, 1;529; CHECK-NEXT: ld.param.b64 %rd3, [generic_acq_rel_sys_param_2];530; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;531; CHECK-NEXT: ld.param.b64 %rd4, [generic_acq_rel_sys_param_3];532; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];533; CHECK-NEXT: ld.param.b64 %rd5, [generic_acq_rel_sys_param_4];534; CHECK-NEXT: add.s16 %rs4, %rs3, 1;535; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;536; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];537; CHECK-NEXT: add.s32 %r2, %r1, 1;538; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;539; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];540; CHECK-NEXT: add.s64 %rd7, %rd6, 1;541; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;542; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];543; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;544; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;545; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];546; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;547; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;548; CHECK-NEXT: ret;549 %a.load = load atomic i8, ptr %a acquire, align 1550 %a.add = add i8 %a.load, 1551 store atomic i8 %a.add, ptr %a release, align 1552 553 %b.load = load atomic i16, ptr %b acquire, align 2554 %b.add = add i16 %b.load, 1555 store atomic i16 %b.add, ptr %b release, align 2556 557 %c.load = load atomic i32, ptr %c acquire, align 4558 %c.add = add i32 %c.load, 1559 store atomic i32 %c.add, ptr %c release, align 4560 561 %d.load = load atomic i64, ptr %d acquire, align 8562 %d.add = add i64 %d.load, 1563 store atomic i64 %d.add, ptr %d release, align 8564 565 %e.load = load atomic float, ptr %e acquire, align 4566 %e.add = fadd float %e.load, 1.567 store atomic float %e.add, ptr %e release, align 4568 569 %f.load = load atomic double, ptr %e acquire, align 8570 %f.add = fadd double %f.load, 1.571 store atomic double %f.add, ptr %e release, align 8572 573 ret void574}575 576; CHECK-LABEL: generic_acq_rel_volatile_sys577define void @generic_acq_rel_volatile_sys(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {578; CHECK-LABEL: generic_acq_rel_volatile_sys(579; CHECK: {580; CHECK-NEXT: .reg .b16 %rs<5>;581; CHECK-NEXT: .reg .b32 %r<5>;582; CHECK-NEXT: .reg .b64 %rd<10>;583; CHECK-EMPTY:584; CHECK-NEXT: // %bb.0:585; CHECK-NEXT: ld.param.b64 %rd1, [generic_acq_rel_volatile_sys_param_0];586; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];587; CHECK-NEXT: ld.param.b64 %rd2, [generic_acq_rel_volatile_sys_param_1];588; CHECK-NEXT: add.s16 %rs2, %rs1, 1;589; CHECK-NEXT: ld.param.b64 %rd3, [generic_acq_rel_volatile_sys_param_2];590; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;591; CHECK-NEXT: ld.param.b64 %rd4, [generic_acq_rel_volatile_sys_param_3];592; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];593; CHECK-NEXT: ld.param.b64 %rd5, [generic_acq_rel_volatile_sys_param_4];594; CHECK-NEXT: add.s16 %rs4, %rs3, 1;595; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;596; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];597; CHECK-NEXT: add.s32 %r2, %r1, 1;598; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;599; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];600; CHECK-NEXT: add.s64 %rd7, %rd6, 1;601; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;602; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];603; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;604; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;605; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];606; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;607; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;608; CHECK-NEXT: ret;609 %a.load = load atomic volatile i8, ptr %a acquire, align 1610 %a.add = add i8 %a.load, 1611 store atomic volatile i8 %a.add, ptr %a release, align 1612 613 %b.load = load atomic volatile i16, ptr %b acquire, align 2614 %b.add = add i16 %b.load, 1615 store atomic volatile i16 %b.add, ptr %b release, align 2616 617 %c.load = load atomic volatile i32, ptr %c acquire, align 4618 %c.add = add i32 %c.load, 1619 store atomic volatile i32 %c.add, ptr %c release, align 4620 621 %d.load = load atomic volatile i64, ptr %d acquire, align 8622 %d.add = add i64 %d.load, 1623 store atomic volatile i64 %d.add, ptr %d release, align 8624 625 %e.load = load atomic volatile float, ptr %e acquire, align 4626 %e.add = fadd float %e.load, 1.627 store atomic volatile float %e.add, ptr %e release, align 4628 629 %f.load = load atomic volatile double, ptr %e acquire, align 8630 %f.add = fadd double %f.load, 1.631 store atomic volatile double %f.add, ptr %e release, align 8632 633 ret void634}635 636; CHECK-LABEL: generic_acq_rel_gpu637define void @generic_acq_rel_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {638; CHECK-LABEL: generic_acq_rel_gpu(639; CHECK: {640; CHECK-NEXT: .reg .b16 %rs<5>;641; CHECK-NEXT: .reg .b32 %r<5>;642; CHECK-NEXT: .reg .b64 %rd<10>;643; CHECK-EMPTY:644; CHECK-NEXT: // %bb.0:645; CHECK-NEXT: ld.param.b64 %rd1, [generic_acq_rel_gpu_param_0];646; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];647; CHECK-NEXT: ld.param.b64 %rd2, [generic_acq_rel_gpu_param_1];648; CHECK-NEXT: add.s16 %rs2, %rs1, 1;649; CHECK-NEXT: ld.param.b64 %rd3, [generic_acq_rel_gpu_param_2];650; CHECK-NEXT: st.release.gpu.b8 [%rd1], %rs2;651; CHECK-NEXT: ld.param.b64 %rd4, [generic_acq_rel_gpu_param_3];652; CHECK-NEXT: ld.acquire.gpu.b16 %rs3, [%rd2];653; CHECK-NEXT: ld.param.b64 %rd5, [generic_acq_rel_gpu_param_4];654; CHECK-NEXT: add.s16 %rs4, %rs3, 1;655; CHECK-NEXT: st.release.gpu.b16 [%rd2], %rs4;656; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd3];657; CHECK-NEXT: add.s32 %r2, %r1, 1;658; CHECK-NEXT: st.release.gpu.b32 [%rd3], %r2;659; CHECK-NEXT: ld.acquire.gpu.b64 %rd6, [%rd4];660; CHECK-NEXT: add.s64 %rd7, %rd6, 1;661; CHECK-NEXT: st.release.gpu.b64 [%rd4], %rd7;662; CHECK-NEXT: ld.acquire.gpu.b32 %r3, [%rd5];663; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;664; CHECK-NEXT: st.release.gpu.b32 [%rd5], %r4;665; CHECK-NEXT: ld.acquire.gpu.b64 %rd8, [%rd5];666; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;667; CHECK-NEXT: st.release.gpu.b64 [%rd5], %rd9;668; CHECK-NEXT: ret;669 %a.load = load atomic i8, ptr %a syncscope("device") acquire, align 1670 %a.add = add i8 %a.load, 1671 store atomic i8 %a.add, ptr %a syncscope("device") release, align 1672 673 %b.load = load atomic i16, ptr %b syncscope("device") acquire, align 2674 %b.add = add i16 %b.load, 1675 store atomic i16 %b.add, ptr %b syncscope("device") release, align 2676 677 %c.load = load atomic i32, ptr %c syncscope("device") acquire, align 4678 %c.add = add i32 %c.load, 1679 store atomic i32 %c.add, ptr %c syncscope("device") release, align 4680 681 %d.load = load atomic i64, ptr %d syncscope("device") acquire, align 8682 %d.add = add i64 %d.load, 1683 store atomic i64 %d.add, ptr %d syncscope("device") release, align 8684 685 %e.load = load atomic float, ptr %e syncscope("device") acquire, align 4686 %e.add = fadd float %e.load, 1.687 store atomic float %e.add, ptr %e syncscope("device") release, align 4688 689 %f.load = load atomic double, ptr %e syncscope("device") acquire, align 8690 %f.add = fadd double %f.load, 1.691 store atomic double %f.add, ptr %e syncscope("device") release, align 8692 693 ret void694}695 696; CHECK-LABEL: generic_acq_rel_volatile_gpu697define void @generic_acq_rel_volatile_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {698; CHECK-LABEL: generic_acq_rel_volatile_gpu(699; CHECK: {700; CHECK-NEXT: .reg .b16 %rs<5>;701; CHECK-NEXT: .reg .b32 %r<5>;702; CHECK-NEXT: .reg .b64 %rd<10>;703; CHECK-EMPTY:704; CHECK-NEXT: // %bb.0:705; CHECK-NEXT: ld.param.b64 %rd1, [generic_acq_rel_volatile_gpu_param_0];706; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];707; CHECK-NEXT: ld.param.b64 %rd2, [generic_acq_rel_volatile_gpu_param_1];708; CHECK-NEXT: add.s16 %rs2, %rs1, 1;709; CHECK-NEXT: ld.param.b64 %rd3, [generic_acq_rel_volatile_gpu_param_2];710; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;711; CHECK-NEXT: ld.param.b64 %rd4, [generic_acq_rel_volatile_gpu_param_3];712; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];713; CHECK-NEXT: ld.param.b64 %rd5, [generic_acq_rel_volatile_gpu_param_4];714; CHECK-NEXT: add.s16 %rs4, %rs3, 1;715; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;716; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];717; CHECK-NEXT: add.s32 %r2, %r1, 1;718; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;719; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];720; CHECK-NEXT: add.s64 %rd7, %rd6, 1;721; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;722; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];723; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;724; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;725; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];726; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;727; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;728; CHECK-NEXT: ret;729 %a.load = load atomic volatile i8, ptr %a syncscope("device") acquire, align 1730 %a.add = add i8 %a.load, 1731 store atomic volatile i8 %a.add, ptr %a syncscope("device") release, align 1732 733 %b.load = load atomic volatile i16, ptr %b syncscope("device") acquire, align 2734 %b.add = add i16 %b.load, 1735 store atomic volatile i16 %b.add, ptr %b syncscope("device") release, align 2736 737 %c.load = load atomic volatile i32, ptr %c syncscope("device") acquire, align 4738 %c.add = add i32 %c.load, 1739 store atomic volatile i32 %c.add, ptr %c syncscope("device") release, align 4740 741 %d.load = load atomic volatile i64, ptr %d syncscope("device") acquire, align 8742 %d.add = add i64 %d.load, 1743 store atomic volatile i64 %d.add, ptr %d syncscope("device") release, align 8744 745 %e.load = load atomic volatile float, ptr %e syncscope("device") acquire, align 4746 %e.add = fadd float %e.load, 1.747 store atomic volatile float %e.add, ptr %e syncscope("device") release, align 4748 749 %f.load = load atomic volatile double, ptr %e syncscope("device") acquire, align 8750 %f.add = fadd double %f.load, 1.751 store atomic volatile double %f.add, ptr %e syncscope("device") release, align 8752 753 ret void754}755 756; CHECK-LABEL: generic_acq_rel_cta757define void @generic_acq_rel_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {758; CHECK-LABEL: generic_acq_rel_cta(759; CHECK: {760; CHECK-NEXT: .reg .b16 %rs<5>;761; CHECK-NEXT: .reg .b32 %r<5>;762; CHECK-NEXT: .reg .b64 %rd<10>;763; CHECK-EMPTY:764; CHECK-NEXT: // %bb.0:765; CHECK-NEXT: ld.param.b64 %rd1, [generic_acq_rel_cta_param_0];766; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];767; CHECK-NEXT: ld.param.b64 %rd2, [generic_acq_rel_cta_param_1];768; CHECK-NEXT: add.s16 %rs2, %rs1, 1;769; CHECK-NEXT: ld.param.b64 %rd3, [generic_acq_rel_cta_param_2];770; CHECK-NEXT: st.release.cta.b8 [%rd1], %rs2;771; CHECK-NEXT: ld.param.b64 %rd4, [generic_acq_rel_cta_param_3];772; CHECK-NEXT: ld.acquire.cta.b16 %rs3, [%rd2];773; CHECK-NEXT: ld.param.b64 %rd5, [generic_acq_rel_cta_param_4];774; CHECK-NEXT: add.s16 %rs4, %rs3, 1;775; CHECK-NEXT: st.release.cta.b16 [%rd2], %rs4;776; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd3];777; CHECK-NEXT: add.s32 %r2, %r1, 1;778; CHECK-NEXT: st.release.cta.b32 [%rd3], %r2;779; CHECK-NEXT: ld.acquire.cta.b64 %rd6, [%rd4];780; CHECK-NEXT: add.s64 %rd7, %rd6, 1;781; CHECK-NEXT: st.release.cta.b64 [%rd4], %rd7;782; CHECK-NEXT: ld.acquire.cta.b32 %r3, [%rd5];783; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;784; CHECK-NEXT: st.release.cta.b32 [%rd5], %r4;785; CHECK-NEXT: ld.acquire.cta.b64 %rd8, [%rd5];786; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;787; CHECK-NEXT: st.release.cta.b64 [%rd5], %rd9;788; CHECK-NEXT: ret;789 %a.load = load atomic i8, ptr %a syncscope("block") acquire, align 1790 %a.add = add i8 %a.load, 1791 store atomic i8 %a.add, ptr %a syncscope("block") release, align 1792 793 %b.load = load atomic i16, ptr %b syncscope("block") acquire, align 2794 %b.add = add i16 %b.load, 1795 store atomic i16 %b.add, ptr %b syncscope("block") release, align 2796 797 %c.load = load atomic i32, ptr %c syncscope("block") acquire, align 4798 %c.add = add i32 %c.load, 1799 store atomic i32 %c.add, ptr %c syncscope("block") release, align 4800 801 %d.load = load atomic i64, ptr %d syncscope("block") acquire, align 8802 %d.add = add i64 %d.load, 1803 store atomic i64 %d.add, ptr %d syncscope("block") release, align 8804 805 %e.load = load atomic float, ptr %e syncscope("block") acquire, align 4806 %e.add = fadd float %e.load, 1.807 store atomic float %e.add, ptr %e syncscope("block") release, align 4808 809 %f.load = load atomic double, ptr %e syncscope("block") acquire, align 8810 %f.add = fadd double %f.load, 1.811 store atomic double %f.add, ptr %e syncscope("block") release, align 8812 813 ret void814}815 816; CHECK-LABEL: generic_acq_rel_volatile_cta817define void @generic_acq_rel_volatile_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {818; CHECK-LABEL: generic_acq_rel_volatile_cta(819; CHECK: {820; CHECK-NEXT: .reg .b16 %rs<5>;821; CHECK-NEXT: .reg .b32 %r<5>;822; CHECK-NEXT: .reg .b64 %rd<10>;823; CHECK-EMPTY:824; CHECK-NEXT: // %bb.0:825; CHECK-NEXT: ld.param.b64 %rd1, [generic_acq_rel_volatile_cta_param_0];826; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];827; CHECK-NEXT: ld.param.b64 %rd2, [generic_acq_rel_volatile_cta_param_1];828; CHECK-NEXT: add.s16 %rs2, %rs1, 1;829; CHECK-NEXT: ld.param.b64 %rd3, [generic_acq_rel_volatile_cta_param_2];830; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;831; CHECK-NEXT: ld.param.b64 %rd4, [generic_acq_rel_volatile_cta_param_3];832; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];833; CHECK-NEXT: ld.param.b64 %rd5, [generic_acq_rel_volatile_cta_param_4];834; CHECK-NEXT: add.s16 %rs4, %rs3, 1;835; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;836; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];837; CHECK-NEXT: add.s32 %r2, %r1, 1;838; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;839; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];840; CHECK-NEXT: add.s64 %rd7, %rd6, 1;841; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;842; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];843; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;844; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;845; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];846; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;847; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;848; CHECK-NEXT: ret;849 %a.load = load atomic volatile i8, ptr %a syncscope("block") acquire, align 1850 %a.add = add i8 %a.load, 1851 store atomic volatile i8 %a.add, ptr %a syncscope("block") release, align 1852 853 %b.load = load atomic volatile i16, ptr %b syncscope("block") acquire, align 2854 %b.add = add i16 %b.load, 1855 store atomic volatile i16 %b.add, ptr %b syncscope("block") release, align 2856 857 %c.load = load atomic volatile i32, ptr %c syncscope("block") acquire, align 4858 %c.add = add i32 %c.load, 1859 store atomic volatile i32 %c.add, ptr %c syncscope("block") release, align 4860 861 %d.load = load atomic volatile i64, ptr %d syncscope("block") acquire, align 8862 %d.add = add i64 %d.load, 1863 store atomic volatile i64 %d.add, ptr %d syncscope("block") release, align 8864 865 %e.load = load atomic volatile float, ptr %e syncscope("block") acquire, align 4866 %e.add = fadd float %e.load, 1.867 store atomic volatile float %e.add, ptr %e syncscope("block") release, align 4868 869 %f.load = load atomic volatile double, ptr %e syncscope("block") acquire, align 8870 %f.add = fadd double %f.load, 1.871 store atomic volatile double %f.add, ptr %e syncscope("block") release, align 8872 873 ret void874}875 876; CHECK-LABEL: generic_sc_sys877define void @generic_sc_sys(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {878; CHECK-LABEL: generic_sc_sys(879; CHECK: {880; CHECK-NEXT: .reg .b16 %rs<5>;881; CHECK-NEXT: .reg .b32 %r<5>;882; CHECK-NEXT: .reg .b64 %rd<10>;883; CHECK-EMPTY:884; CHECK-NEXT: // %bb.0:885; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_sys_param_0];886; CHECK-NEXT: fence.sc.sys;887; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];888; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_sys_param_1];889; CHECK-NEXT: add.s16 %rs2, %rs1, 1;890; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_sys_param_2];891; CHECK-NEXT: fence.sc.sys;892; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;893; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_sys_param_3];894; CHECK-NEXT: fence.sc.sys;895; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];896; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_sys_param_4];897; CHECK-NEXT: add.s16 %rs4, %rs3, 1;898; CHECK-NEXT: fence.sc.sys;899; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;900; CHECK-NEXT: fence.sc.sys;901; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];902; CHECK-NEXT: add.s32 %r2, %r1, 1;903; CHECK-NEXT: fence.sc.sys;904; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;905; CHECK-NEXT: fence.sc.sys;906; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];907; CHECK-NEXT: add.s64 %rd7, %rd6, 1;908; CHECK-NEXT: fence.sc.sys;909; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;910; CHECK-NEXT: fence.sc.sys;911; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];912; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;913; CHECK-NEXT: fence.sc.sys;914; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;915; CHECK-NEXT: fence.sc.sys;916; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];917; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;918; CHECK-NEXT: fence.sc.sys;919; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;920; CHECK-NEXT: ret;921 %a.load = load atomic i8, ptr %a seq_cst, align 1922 %a.add = add i8 %a.load, 1923 store atomic i8 %a.add, ptr %a seq_cst, align 1924 925 %b.load = load atomic i16, ptr %b seq_cst, align 2926 %b.add = add i16 %b.load, 1927 store atomic i16 %b.add, ptr %b seq_cst, align 2928 929 %c.load = load atomic i32, ptr %c seq_cst, align 4930 %c.add = add i32 %c.load, 1931 store atomic i32 %c.add, ptr %c seq_cst, align 4932 933 %d.load = load atomic i64, ptr %d seq_cst, align 8934 %d.add = add i64 %d.load, 1935 store atomic i64 %d.add, ptr %d seq_cst, align 8936 937 %e.load = load atomic float, ptr %e seq_cst, align 4938 %e.add = fadd float %e.load, 1.939 store atomic float %e.add, ptr %e seq_cst, align 4940 941 %f.load = load atomic double, ptr %e seq_cst, align 8942 %f.add = fadd double %f.load, 1.943 store atomic double %f.add, ptr %e seq_cst, align 8944 945 ret void946}947 948; CHECK-LABEL: generic_sc_volatile_sys949define void @generic_sc_volatile_sys(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {950; CHECK-LABEL: generic_sc_volatile_sys(951; CHECK: {952; CHECK-NEXT: .reg .b16 %rs<5>;953; CHECK-NEXT: .reg .b32 %r<5>;954; CHECK-NEXT: .reg .b64 %rd<10>;955; CHECK-EMPTY:956; CHECK-NEXT: // %bb.0:957; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_volatile_sys_param_0];958; CHECK-NEXT: fence.sc.sys;959; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];960; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_volatile_sys_param_1];961; CHECK-NEXT: add.s16 %rs2, %rs1, 1;962; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_volatile_sys_param_2];963; CHECK-NEXT: fence.sc.sys;964; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;965; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_volatile_sys_param_3];966; CHECK-NEXT: fence.sc.sys;967; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];968; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_volatile_sys_param_4];969; CHECK-NEXT: add.s16 %rs4, %rs3, 1;970; CHECK-NEXT: fence.sc.sys;971; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;972; CHECK-NEXT: fence.sc.sys;973; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];974; CHECK-NEXT: add.s32 %r2, %r1, 1;975; CHECK-NEXT: fence.sc.sys;976; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;977; CHECK-NEXT: fence.sc.sys;978; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];979; CHECK-NEXT: add.s64 %rd7, %rd6, 1;980; CHECK-NEXT: fence.sc.sys;981; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;982; CHECK-NEXT: fence.sc.sys;983; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];984; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;985; CHECK-NEXT: fence.sc.sys;986; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;987; CHECK-NEXT: fence.sc.sys;988; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];989; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;990; CHECK-NEXT: fence.sc.sys;991; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;992; CHECK-NEXT: ret;993 %a.load = load atomic volatile i8, ptr %a seq_cst, align 1994 %a.add = add i8 %a.load, 1995 store atomic volatile i8 %a.add, ptr %a seq_cst, align 1996 997 %b.load = load atomic volatile i16, ptr %b seq_cst, align 2998 %b.add = add i16 %b.load, 1999 store atomic volatile i16 %b.add, ptr %b seq_cst, align 21000 1001 %c.load = load atomic volatile i32, ptr %c seq_cst, align 41002 %c.add = add i32 %c.load, 11003 store atomic volatile i32 %c.add, ptr %c seq_cst, align 41004 1005 %d.load = load atomic volatile i64, ptr %d seq_cst, align 81006 %d.add = add i64 %d.load, 11007 store atomic volatile i64 %d.add, ptr %d seq_cst, align 81008 1009 %e.load = load atomic volatile float, ptr %e seq_cst, align 41010 %e.add = fadd float %e.load, 1.1011 store atomic volatile float %e.add, ptr %e seq_cst, align 41012 1013 %f.load = load atomic volatile double, ptr %e seq_cst, align 81014 %f.add = fadd double %f.load, 1.1015 store atomic volatile double %f.add, ptr %e seq_cst, align 81016 1017 ret void1018}1019 1020; CHECK-LABEL: generic_sc_gpu1021define void @generic_sc_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {1022; CHECK-LABEL: generic_sc_gpu(1023; CHECK: {1024; CHECK-NEXT: .reg .b16 %rs<5>;1025; CHECK-NEXT: .reg .b32 %r<5>;1026; CHECK-NEXT: .reg .b64 %rd<10>;1027; CHECK-EMPTY:1028; CHECK-NEXT: // %bb.0:1029; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_gpu_param_0];1030; CHECK-NEXT: fence.sc.gpu;1031; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];1032; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_gpu_param_1];1033; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1034; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_gpu_param_2];1035; CHECK-NEXT: fence.sc.gpu;1036; CHECK-NEXT: st.release.gpu.b8 [%rd1], %rs2;1037; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_gpu_param_3];1038; CHECK-NEXT: fence.sc.gpu;1039; CHECK-NEXT: ld.acquire.gpu.b16 %rs3, [%rd2];1040; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_gpu_param_4];1041; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1042; CHECK-NEXT: fence.sc.gpu;1043; CHECK-NEXT: st.release.gpu.b16 [%rd2], %rs4;1044; CHECK-NEXT: fence.sc.gpu;1045; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd3];1046; CHECK-NEXT: add.s32 %r2, %r1, 1;1047; CHECK-NEXT: fence.sc.gpu;1048; CHECK-NEXT: st.release.gpu.b32 [%rd3], %r2;1049; CHECK-NEXT: fence.sc.gpu;1050; CHECK-NEXT: ld.acquire.gpu.b64 %rd6, [%rd4];1051; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1052; CHECK-NEXT: fence.sc.gpu;1053; CHECK-NEXT: st.release.gpu.b64 [%rd4], %rd7;1054; CHECK-NEXT: fence.sc.gpu;1055; CHECK-NEXT: ld.acquire.gpu.b32 %r3, [%rd5];1056; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1057; CHECK-NEXT: fence.sc.gpu;1058; CHECK-NEXT: st.release.gpu.b32 [%rd5], %r4;1059; CHECK-NEXT: fence.sc.gpu;1060; CHECK-NEXT: ld.acquire.gpu.b64 %rd8, [%rd5];1061; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1062; CHECK-NEXT: fence.sc.gpu;1063; CHECK-NEXT: st.release.gpu.b64 [%rd5], %rd9;1064; CHECK-NEXT: ret;1065 %a.load = load atomic i8, ptr %a syncscope("device") seq_cst, align 11066 %a.add = add i8 %a.load, 11067 store atomic i8 %a.add, ptr %a syncscope("device") seq_cst, align 11068 1069 %b.load = load atomic i16, ptr %b syncscope("device") seq_cst, align 21070 %b.add = add i16 %b.load, 11071 store atomic i16 %b.add, ptr %b syncscope("device") seq_cst, align 21072 1073 %c.load = load atomic i32, ptr %c syncscope("device") seq_cst, align 41074 %c.add = add i32 %c.load, 11075 store atomic i32 %c.add, ptr %c syncscope("device") seq_cst, align 41076 1077 %d.load = load atomic i64, ptr %d syncscope("device") seq_cst, align 81078 %d.add = add i64 %d.load, 11079 store atomic i64 %d.add, ptr %d syncscope("device") seq_cst, align 81080 1081 %e.load = load atomic float, ptr %e syncscope("device") seq_cst, align 41082 %e.add = fadd float %e.load, 1.1083 store atomic float %e.add, ptr %e syncscope("device") seq_cst, align 41084 1085 %f.load = load atomic double, ptr %e syncscope("device") seq_cst, align 81086 %f.add = fadd double %f.load, 1.1087 store atomic double %f.add, ptr %e syncscope("device") seq_cst, align 81088 1089 ret void1090}1091 1092; CHECK-LABEL: generic_sc_volatile_gpu1093define void @generic_sc_volatile_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {1094; CHECK-LABEL: generic_sc_volatile_gpu(1095; CHECK: {1096; CHECK-NEXT: .reg .b16 %rs<5>;1097; CHECK-NEXT: .reg .b32 %r<5>;1098; CHECK-NEXT: .reg .b64 %rd<10>;1099; CHECK-EMPTY:1100; CHECK-NEXT: // %bb.0:1101; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_volatile_gpu_param_0];1102; CHECK-NEXT: fence.sc.sys;1103; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];1104; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_volatile_gpu_param_1];1105; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1106; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_volatile_gpu_param_2];1107; CHECK-NEXT: fence.sc.sys;1108; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;1109; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_volatile_gpu_param_3];1110; CHECK-NEXT: fence.sc.sys;1111; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];1112; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_volatile_gpu_param_4];1113; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1114; CHECK-NEXT: fence.sc.sys;1115; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;1116; CHECK-NEXT: fence.sc.sys;1117; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];1118; CHECK-NEXT: add.s32 %r2, %r1, 1;1119; CHECK-NEXT: fence.sc.sys;1120; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;1121; CHECK-NEXT: fence.sc.sys;1122; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];1123; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1124; CHECK-NEXT: fence.sc.sys;1125; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;1126; CHECK-NEXT: fence.sc.sys;1127; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];1128; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1129; CHECK-NEXT: fence.sc.sys;1130; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;1131; CHECK-NEXT: fence.sc.sys;1132; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];1133; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1134; CHECK-NEXT: fence.sc.sys;1135; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;1136; CHECK-NEXT: ret;1137 %a.load = load atomic volatile i8, ptr %a syncscope("device") seq_cst, align 11138 %a.add = add i8 %a.load, 11139 store atomic volatile i8 %a.add, ptr %a syncscope("device") seq_cst, align 11140 1141 %b.load = load atomic volatile i16, ptr %b syncscope("device") seq_cst, align 21142 %b.add = add i16 %b.load, 11143 store atomic volatile i16 %b.add, ptr %b syncscope("device") seq_cst, align 21144 1145 %c.load = load atomic volatile i32, ptr %c syncscope("device") seq_cst, align 41146 %c.add = add i32 %c.load, 11147 store atomic volatile i32 %c.add, ptr %c syncscope("device") seq_cst, align 41148 1149 %d.load = load atomic volatile i64, ptr %d syncscope("device") seq_cst, align 81150 %d.add = add i64 %d.load, 11151 store atomic volatile i64 %d.add, ptr %d syncscope("device") seq_cst, align 81152 1153 %e.load = load atomic volatile float, ptr %e syncscope("device") seq_cst, align 41154 %e.add = fadd float %e.load, 1.1155 store atomic volatile float %e.add, ptr %e syncscope("device") seq_cst, align 41156 1157 %f.load = load atomic volatile double, ptr %e syncscope("device") seq_cst, align 81158 %f.add = fadd double %f.load, 1.1159 store atomic volatile double %f.add, ptr %e syncscope("device") seq_cst, align 81160 1161 ret void1162}1163 1164; CHECK-LABEL: generic_sc_cta1165define void @generic_sc_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {1166; CHECK-LABEL: generic_sc_cta(1167; CHECK: {1168; CHECK-NEXT: .reg .b16 %rs<5>;1169; CHECK-NEXT: .reg .b32 %r<5>;1170; CHECK-NEXT: .reg .b64 %rd<10>;1171; CHECK-EMPTY:1172; CHECK-NEXT: // %bb.0:1173; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_cta_param_0];1174; CHECK-NEXT: fence.sc.cta;1175; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];1176; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_cta_param_1];1177; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1178; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_cta_param_2];1179; CHECK-NEXT: fence.sc.cta;1180; CHECK-NEXT: st.release.cta.b8 [%rd1], %rs2;1181; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_cta_param_3];1182; CHECK-NEXT: fence.sc.cta;1183; CHECK-NEXT: ld.acquire.cta.b16 %rs3, [%rd2];1184; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_cta_param_4];1185; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1186; CHECK-NEXT: fence.sc.cta;1187; CHECK-NEXT: st.release.cta.b16 [%rd2], %rs4;1188; CHECK-NEXT: fence.sc.cta;1189; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd3];1190; CHECK-NEXT: add.s32 %r2, %r1, 1;1191; CHECK-NEXT: fence.sc.cta;1192; CHECK-NEXT: st.release.cta.b32 [%rd3], %r2;1193; CHECK-NEXT: fence.sc.cta;1194; CHECK-NEXT: ld.acquire.cta.b64 %rd6, [%rd4];1195; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1196; CHECK-NEXT: fence.sc.cta;1197; CHECK-NEXT: st.release.cta.b64 [%rd4], %rd7;1198; CHECK-NEXT: fence.sc.cta;1199; CHECK-NEXT: ld.acquire.cta.b32 %r3, [%rd5];1200; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1201; CHECK-NEXT: fence.sc.cta;1202; CHECK-NEXT: st.release.cta.b32 [%rd5], %r4;1203; CHECK-NEXT: fence.sc.cta;1204; CHECK-NEXT: ld.acquire.cta.b64 %rd8, [%rd5];1205; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1206; CHECK-NEXT: fence.sc.cta;1207; CHECK-NEXT: st.release.cta.b64 [%rd5], %rd9;1208; CHECK-NEXT: ret;1209 %a.load = load atomic i8, ptr %a syncscope("block") seq_cst, align 11210 %a.add = add i8 %a.load, 11211 store atomic i8 %a.add, ptr %a syncscope("block") seq_cst, align 11212 1213 %b.load = load atomic i16, ptr %b syncscope("block") seq_cst, align 21214 %b.add = add i16 %b.load, 11215 store atomic i16 %b.add, ptr %b syncscope("block") seq_cst, align 21216 1217 %c.load = load atomic i32, ptr %c syncscope("block") seq_cst, align 41218 %c.add = add i32 %c.load, 11219 store atomic i32 %c.add, ptr %c syncscope("block") seq_cst, align 41220 1221 %d.load = load atomic i64, ptr %d syncscope("block") seq_cst, align 81222 %d.add = add i64 %d.load, 11223 store atomic i64 %d.add, ptr %d syncscope("block") seq_cst, align 81224 1225 %e.load = load atomic float, ptr %e syncscope("block") seq_cst, align 41226 %e.add = fadd float %e.load, 1.1227 store atomic float %e.add, ptr %e syncscope("block") seq_cst, align 41228 1229 %f.load = load atomic double, ptr %e syncscope("block") seq_cst, align 81230 %f.add = fadd double %f.load, 1.1231 store atomic double %f.add, ptr %e syncscope("block") seq_cst, align 81232 1233 ret void1234}1235 1236; CHECK-LABEL: generic_sc_volatile_cta1237define void @generic_sc_volatile_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unnamed_addr {1238; CHECK-LABEL: generic_sc_volatile_cta(1239; CHECK: {1240; CHECK-NEXT: .reg .b16 %rs<5>;1241; CHECK-NEXT: .reg .b32 %r<5>;1242; CHECK-NEXT: .reg .b64 %rd<10>;1243; CHECK-EMPTY:1244; CHECK-NEXT: // %bb.0:1245; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_volatile_cta_param_0];1246; CHECK-NEXT: fence.sc.sys;1247; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];1248; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_volatile_cta_param_1];1249; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1250; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_volatile_cta_param_2];1251; CHECK-NEXT: fence.sc.sys;1252; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;1253; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_volatile_cta_param_3];1254; CHECK-NEXT: fence.sc.sys;1255; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];1256; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_volatile_cta_param_4];1257; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1258; CHECK-NEXT: fence.sc.sys;1259; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;1260; CHECK-NEXT: fence.sc.sys;1261; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];1262; CHECK-NEXT: add.s32 %r2, %r1, 1;1263; CHECK-NEXT: fence.sc.sys;1264; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;1265; CHECK-NEXT: fence.sc.sys;1266; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];1267; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1268; CHECK-NEXT: fence.sc.sys;1269; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;1270; CHECK-NEXT: fence.sc.sys;1271; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];1272; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1273; CHECK-NEXT: fence.sc.sys;1274; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;1275; CHECK-NEXT: fence.sc.sys;1276; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];1277; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1278; CHECK-NEXT: fence.sc.sys;1279; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;1280; CHECK-NEXT: ret;1281 %a.load = load atomic volatile i8, ptr %a syncscope("block") seq_cst, align 11282 %a.add = add i8 %a.load, 11283 store atomic volatile i8 %a.add, ptr %a syncscope("block") seq_cst, align 11284 1285 %b.load = load atomic volatile i16, ptr %b syncscope("block") seq_cst, align 21286 %b.add = add i16 %b.load, 11287 store atomic volatile i16 %b.add, ptr %b syncscope("block") seq_cst, align 21288 1289 %c.load = load atomic volatile i32, ptr %c syncscope("block") seq_cst, align 41290 %c.add = add i32 %c.load, 11291 store atomic volatile i32 %c.add, ptr %c syncscope("block") seq_cst, align 41292 1293 %d.load = load atomic volatile i64, ptr %d syncscope("block") seq_cst, align 81294 %d.add = add i64 %d.load, 11295 store atomic volatile i64 %d.add, ptr %d syncscope("block") seq_cst, align 81296 1297 %e.load = load atomic volatile float, ptr %e syncscope("block") seq_cst, align 41298 %e.add = fadd float %e.load, 1.1299 store atomic volatile float %e.add, ptr %e syncscope("block") seq_cst, align 41300 1301 %f.load = load atomic volatile double, ptr %e syncscope("block") seq_cst, align 81302 %f.add = fadd double %f.load, 1.1303 store atomic volatile double %f.add, ptr %e syncscope("block") seq_cst, align 81304 1305 ret void1306}1307 1308;; global statespace1309 1310; CHECK-LABEL: global_unordered_gpu1311define void @global_unordered_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1312; CHECK-LABEL: global_unordered_gpu(1313; CHECK: {1314; CHECK-NEXT: .reg .b16 %rs<5>;1315; CHECK-NEXT: .reg .b32 %r<5>;1316; CHECK-NEXT: .reg .b64 %rd<10>;1317; CHECK-EMPTY:1318; CHECK-NEXT: // %bb.0:1319; CHECK-NEXT: ld.param.b64 %rd1, [global_unordered_gpu_param_0];1320; CHECK-NEXT: ld.relaxed.gpu.global.b8 %rs1, [%rd1];1321; CHECK-NEXT: ld.param.b64 %rd2, [global_unordered_gpu_param_1];1322; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1323; CHECK-NEXT: ld.param.b64 %rd3, [global_unordered_gpu_param_2];1324; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs2;1325; CHECK-NEXT: ld.param.b64 %rd4, [global_unordered_gpu_param_3];1326; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs3, [%rd2];1327; CHECK-NEXT: ld.param.b64 %rd5, [global_unordered_gpu_param_4];1328; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1329; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd2], %rs4;1330; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd3];1331; CHECK-NEXT: add.s32 %r2, %r1, 1;1332; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd3], %r2;1333; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd6, [%rd4];1334; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1335; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd4], %rd7;1336; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r3, [%rd5];1337; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1338; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd5], %r4;1339; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd8, [%rd5];1340; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1341; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd5], %rd9;1342; CHECK-NEXT: ret;1343 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("device") unordered, align 11344 %a.add = add i8 %a.load, 11345 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("device") unordered, align 11346 1347 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("device") unordered, align 21348 %b.add = add i16 %b.load, 11349 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("device") unordered, align 21350 1351 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("device") unordered, align 41352 %c.add = add i32 %c.load, 11353 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("device") unordered, align 41354 1355 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("device") unordered, align 81356 %d.add = add i64 %d.load, 11357 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("device") unordered, align 81358 1359 %e.load = load atomic float, ptr addrspace(1) %e syncscope("device") unordered, align 41360 %e.add = fadd float %e.load, 1.1361 store atomic float %e.add, ptr addrspace(1) %e syncscope("device") unordered, align 41362 1363 %f.load = load atomic double, ptr addrspace(1) %e syncscope("device") unordered, align 81364 %f.add = fadd double %f.load, 1.1365 store atomic double %f.add, ptr addrspace(1) %e syncscope("device") unordered, align 81366 1367 ret void1368}1369 1370; CHECK-LABEL: global_unordered_volatile_gpu1371define void @global_unordered_volatile_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1372; CHECK-LABEL: global_unordered_volatile_gpu(1373; CHECK: {1374; CHECK-NEXT: .reg .b16 %rs<5>;1375; CHECK-NEXT: .reg .b32 %r<5>;1376; CHECK-NEXT: .reg .b64 %rd<10>;1377; CHECK-EMPTY:1378; CHECK-NEXT: // %bb.0:1379; CHECK-NEXT: ld.param.b64 %rd1, [global_unordered_volatile_gpu_param_0];1380; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];1381; CHECK-NEXT: ld.param.b64 %rd2, [global_unordered_volatile_gpu_param_1];1382; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1383; CHECK-NEXT: ld.param.b64 %rd3, [global_unordered_volatile_gpu_param_2];1384; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs2;1385; CHECK-NEXT: ld.param.b64 %rd4, [global_unordered_volatile_gpu_param_3];1386; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs3, [%rd2];1387; CHECK-NEXT: ld.param.b64 %rd5, [global_unordered_volatile_gpu_param_4];1388; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1389; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd2], %rs4;1390; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd3];1391; CHECK-NEXT: add.s32 %r2, %r1, 1;1392; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd3], %r2;1393; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd6, [%rd4];1394; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1395; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd4], %rd7;1396; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r3, [%rd5];1397; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1398; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd5], %r4;1399; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd8, [%rd5];1400; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1401; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd5], %rd9;1402; CHECK-NEXT: ret;1403 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("device") unordered, align 11404 %a.add = add i8 %a.load, 11405 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("device") unordered, align 11406 1407 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("device") unordered, align 21408 %b.add = add i16 %b.load, 11409 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("device") unordered, align 21410 1411 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("device") unordered, align 41412 %c.add = add i32 %c.load, 11413 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("device") unordered, align 41414 1415 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("device") unordered, align 81416 %d.add = add i64 %d.load, 11417 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("device") unordered, align 81418 1419 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("device") unordered, align 41420 %e.add = fadd float %e.load, 1.1421 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("device") unordered, align 41422 1423 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("device") unordered, align 81424 %f.add = fadd double %f.load, 1.1425 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("device") unordered, align 81426 1427 ret void1428}1429 1430; CHECK-LABEL: global_unordered_cta1431define void @global_unordered_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1432; CHECK-LABEL: global_unordered_cta(1433; CHECK: {1434; CHECK-NEXT: .reg .b16 %rs<5>;1435; CHECK-NEXT: .reg .b32 %r<5>;1436; CHECK-NEXT: .reg .b64 %rd<10>;1437; CHECK-EMPTY:1438; CHECK-NEXT: // %bb.0:1439; CHECK-NEXT: ld.param.b64 %rd1, [global_unordered_cta_param_0];1440; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];1441; CHECK-NEXT: ld.param.b64 %rd2, [global_unordered_cta_param_1];1442; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1443; CHECK-NEXT: ld.param.b64 %rd3, [global_unordered_cta_param_2];1444; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs2;1445; CHECK-NEXT: ld.param.b64 %rd4, [global_unordered_cta_param_3];1446; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs3, [%rd2];1447; CHECK-NEXT: ld.param.b64 %rd5, [global_unordered_cta_param_4];1448; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1449; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd2], %rs4;1450; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd3];1451; CHECK-NEXT: add.s32 %r2, %r1, 1;1452; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd3], %r2;1453; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd4];1454; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1455; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd4], %rd7;1456; CHECK-NEXT: ld.relaxed.cta.global.b32 %r3, [%rd5];1457; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1458; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd5], %r4;1459; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];1460; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1461; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd5], %rd9;1462; CHECK-NEXT: ret;1463 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("block") unordered, align 11464 %a.add = add i8 %a.load, 11465 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("block") unordered, align 11466 1467 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("block") unordered, align 21468 %b.add = add i16 %b.load, 11469 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("block") unordered, align 21470 1471 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("block") unordered, align 41472 %c.add = add i32 %c.load, 11473 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("block") unordered, align 41474 1475 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("block") unordered, align 81476 %d.add = add i64 %d.load, 11477 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("block") unordered, align 81478 1479 %e.load = load atomic float, ptr addrspace(1) %e syncscope("block") unordered, align 41480 %e.add = fadd float %e.load, 1.1481 store atomic float %e.add, ptr addrspace(1) %e syncscope("block") unordered, align 41482 1483 %f.load = load atomic double, ptr addrspace(1) %e syncscope("block") unordered, align 81484 %f.add = fadd double %f.load, 1.1485 store atomic double %f.add, ptr addrspace(1) %e syncscope("block") unordered, align 81486 1487 ret void1488}1489 1490; CHECK-LABEL: global_unordered_volatile_cta1491define void @global_unordered_volatile_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1492; CHECK-LABEL: global_unordered_volatile_cta(1493; CHECK: {1494; CHECK-NEXT: .reg .b16 %rs<5>;1495; CHECK-NEXT: .reg .b32 %r<5>;1496; CHECK-NEXT: .reg .b64 %rd<10>;1497; CHECK-EMPTY:1498; CHECK-NEXT: // %bb.0:1499; CHECK-NEXT: ld.param.b64 %rd1, [global_unordered_volatile_cta_param_0];1500; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];1501; CHECK-NEXT: ld.param.b64 %rd2, [global_unordered_volatile_cta_param_1];1502; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1503; CHECK-NEXT: ld.param.b64 %rd3, [global_unordered_volatile_cta_param_2];1504; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs2;1505; CHECK-NEXT: ld.param.b64 %rd4, [global_unordered_volatile_cta_param_3];1506; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs3, [%rd2];1507; CHECK-NEXT: ld.param.b64 %rd5, [global_unordered_volatile_cta_param_4];1508; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1509; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd2], %rs4;1510; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd3];1511; CHECK-NEXT: add.s32 %r2, %r1, 1;1512; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd3], %r2;1513; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd6, [%rd4];1514; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1515; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd4], %rd7;1516; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r3, [%rd5];1517; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1518; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd5], %r4;1519; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd8, [%rd5];1520; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1521; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd5], %rd9;1522; CHECK-NEXT: ret;1523 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("block") unordered, align 11524 %a.add = add i8 %a.load, 11525 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("block") unordered, align 11526 1527 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("block") unordered, align 21528 %b.add = add i16 %b.load, 11529 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("block") unordered, align 21530 1531 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("block") unordered, align 41532 %c.add = add i32 %c.load, 11533 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("block") unordered, align 41534 1535 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("block") unordered, align 81536 %d.add = add i64 %d.load, 11537 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("block") unordered, align 81538 1539 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("block") unordered, align 41540 %e.add = fadd float %e.load, 1.1541 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("block") unordered, align 41542 1543 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("block") unordered, align 81544 %f.add = fadd double %f.load, 1.1545 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("block") unordered, align 81546 1547 ret void1548}1549 1550; CHECK-LABEL: global_monotonic_gpu1551define void @global_monotonic_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1552; CHECK-LABEL: global_monotonic_gpu(1553; CHECK: {1554; CHECK-NEXT: .reg .b16 %rs<5>;1555; CHECK-NEXT: .reg .b32 %r<5>;1556; CHECK-NEXT: .reg .b64 %rd<10>;1557; CHECK-EMPTY:1558; CHECK-NEXT: // %bb.0:1559; CHECK-NEXT: ld.param.b64 %rd1, [global_monotonic_gpu_param_0];1560; CHECK-NEXT: ld.relaxed.gpu.global.b8 %rs1, [%rd1];1561; CHECK-NEXT: ld.param.b64 %rd2, [global_monotonic_gpu_param_1];1562; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1563; CHECK-NEXT: ld.param.b64 %rd3, [global_monotonic_gpu_param_2];1564; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs2;1565; CHECK-NEXT: ld.param.b64 %rd4, [global_monotonic_gpu_param_3];1566; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs3, [%rd2];1567; CHECK-NEXT: ld.param.b64 %rd5, [global_monotonic_gpu_param_4];1568; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1569; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd2], %rs4;1570; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd3];1571; CHECK-NEXT: add.s32 %r2, %r1, 1;1572; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd3], %r2;1573; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd6, [%rd4];1574; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1575; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd4], %rd7;1576; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r3, [%rd5];1577; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1578; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd5], %r4;1579; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd8, [%rd5];1580; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1581; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd5], %rd9;1582; CHECK-NEXT: ret;1583 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("device") monotonic, align 11584 %a.add = add i8 %a.load, 11585 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("device") monotonic, align 11586 1587 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("device") monotonic, align 21588 %b.add = add i16 %b.load, 11589 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("device") monotonic, align 21590 1591 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("device") monotonic, align 41592 %c.add = add i32 %c.load, 11593 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("device") monotonic, align 41594 1595 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("device") monotonic, align 81596 %d.add = add i64 %d.load, 11597 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("device") monotonic, align 81598 1599 %e.load = load atomic float, ptr addrspace(1) %e syncscope("device") monotonic, align 41600 %e.add = fadd float %e.load, 1.1601 store atomic float %e.add, ptr addrspace(1) %e syncscope("device") monotonic, align 41602 1603 %f.load = load atomic double, ptr addrspace(1) %e syncscope("device") monotonic, align 81604 %f.add = fadd double %f.load, 1.1605 store atomic double %f.add, ptr addrspace(1) %e syncscope("device") monotonic, align 81606 1607 ret void1608}1609 1610; CHECK-LABEL: global_monotonic_volatile_gpu1611define void @global_monotonic_volatile_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1612; CHECK-LABEL: global_monotonic_volatile_gpu(1613; CHECK: {1614; CHECK-NEXT: .reg .b16 %rs<5>;1615; CHECK-NEXT: .reg .b32 %r<5>;1616; CHECK-NEXT: .reg .b64 %rd<10>;1617; CHECK-EMPTY:1618; CHECK-NEXT: // %bb.0:1619; CHECK-NEXT: ld.param.b64 %rd1, [global_monotonic_volatile_gpu_param_0];1620; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];1621; CHECK-NEXT: ld.param.b64 %rd2, [global_monotonic_volatile_gpu_param_1];1622; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1623; CHECK-NEXT: ld.param.b64 %rd3, [global_monotonic_volatile_gpu_param_2];1624; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs2;1625; CHECK-NEXT: ld.param.b64 %rd4, [global_monotonic_volatile_gpu_param_3];1626; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs3, [%rd2];1627; CHECK-NEXT: ld.param.b64 %rd5, [global_monotonic_volatile_gpu_param_4];1628; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1629; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd2], %rs4;1630; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd3];1631; CHECK-NEXT: add.s32 %r2, %r1, 1;1632; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd3], %r2;1633; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd6, [%rd4];1634; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1635; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd4], %rd7;1636; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r3, [%rd5];1637; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1638; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd5], %r4;1639; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd8, [%rd5];1640; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1641; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd5], %rd9;1642; CHECK-NEXT: ret;1643 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("device") monotonic, align 11644 %a.add = add i8 %a.load, 11645 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("device") monotonic, align 11646 1647 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("device") monotonic, align 21648 %b.add = add i16 %b.load, 11649 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("device") monotonic, align 21650 1651 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("device") monotonic, align 41652 %c.add = add i32 %c.load, 11653 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("device") monotonic, align 41654 1655 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("device") monotonic, align 81656 %d.add = add i64 %d.load, 11657 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("device") monotonic, align 81658 1659 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("device") monotonic, align 41660 %e.add = fadd float %e.load, 1.1661 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("device") monotonic, align 41662 1663 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("device") monotonic, align 81664 %f.add = fadd double %f.load, 1.1665 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("device") monotonic, align 81666 1667 ret void1668}1669 1670; CHECK-LABEL: global_monotonic_cta1671define void @global_monotonic_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1672; CHECK-LABEL: global_monotonic_cta(1673; CHECK: {1674; CHECK-NEXT: .reg .b16 %rs<5>;1675; CHECK-NEXT: .reg .b32 %r<5>;1676; CHECK-NEXT: .reg .b64 %rd<10>;1677; CHECK-EMPTY:1678; CHECK-NEXT: // %bb.0:1679; CHECK-NEXT: ld.param.b64 %rd1, [global_monotonic_cta_param_0];1680; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];1681; CHECK-NEXT: ld.param.b64 %rd2, [global_monotonic_cta_param_1];1682; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1683; CHECK-NEXT: ld.param.b64 %rd3, [global_monotonic_cta_param_2];1684; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs2;1685; CHECK-NEXT: ld.param.b64 %rd4, [global_monotonic_cta_param_3];1686; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs3, [%rd2];1687; CHECK-NEXT: ld.param.b64 %rd5, [global_monotonic_cta_param_4];1688; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1689; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd2], %rs4;1690; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd3];1691; CHECK-NEXT: add.s32 %r2, %r1, 1;1692; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd3], %r2;1693; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd4];1694; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1695; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd4], %rd7;1696; CHECK-NEXT: ld.relaxed.cta.global.b32 %r3, [%rd5];1697; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1698; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd5], %r4;1699; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd8, [%rd5];1700; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1701; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd5], %rd9;1702; CHECK-NEXT: ret;1703 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("block") monotonic, align 11704 %a.add = add i8 %a.load, 11705 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("block") monotonic, align 11706 1707 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("block") monotonic, align 21708 %b.add = add i16 %b.load, 11709 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("block") monotonic, align 21710 1711 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("block") monotonic, align 41712 %c.add = add i32 %c.load, 11713 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("block") monotonic, align 41714 1715 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("block") monotonic, align 81716 %d.add = add i64 %d.load, 11717 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("block") monotonic, align 81718 1719 %e.load = load atomic float, ptr addrspace(1) %e syncscope("block") monotonic, align 41720 %e.add = fadd float %e.load, 1.1721 store atomic float %e.add, ptr addrspace(1) %e syncscope("block") monotonic, align 41722 1723 %f.load = load atomic double, ptr addrspace(1) %e syncscope("block") monotonic, align 81724 %f.add = fadd double %f.load, 1.1725 store atomic double %f.add, ptr addrspace(1) %e syncscope("block") monotonic, align 81726 1727 ret void1728}1729 1730; CHECK-LABEL: global_monotonic_volatile_cta1731define void @global_monotonic_volatile_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1732; CHECK-LABEL: global_monotonic_volatile_cta(1733; CHECK: {1734; CHECK-NEXT: .reg .b16 %rs<5>;1735; CHECK-NEXT: .reg .b32 %r<5>;1736; CHECK-NEXT: .reg .b64 %rd<10>;1737; CHECK-EMPTY:1738; CHECK-NEXT: // %bb.0:1739; CHECK-NEXT: ld.param.b64 %rd1, [global_monotonic_volatile_cta_param_0];1740; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];1741; CHECK-NEXT: ld.param.b64 %rd2, [global_monotonic_volatile_cta_param_1];1742; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1743; CHECK-NEXT: ld.param.b64 %rd3, [global_monotonic_volatile_cta_param_2];1744; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs2;1745; CHECK-NEXT: ld.param.b64 %rd4, [global_monotonic_volatile_cta_param_3];1746; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs3, [%rd2];1747; CHECK-NEXT: ld.param.b64 %rd5, [global_monotonic_volatile_cta_param_4];1748; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1749; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd2], %rs4;1750; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd3];1751; CHECK-NEXT: add.s32 %r2, %r1, 1;1752; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd3], %r2;1753; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd6, [%rd4];1754; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1755; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd4], %rd7;1756; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r3, [%rd5];1757; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1758; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd5], %r4;1759; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd8, [%rd5];1760; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1761; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd5], %rd9;1762; CHECK-NEXT: ret;1763 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("block") monotonic, align 11764 %a.add = add i8 %a.load, 11765 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("block") monotonic, align 11766 1767 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("block") monotonic, align 21768 %b.add = add i16 %b.load, 11769 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("block") monotonic, align 21770 1771 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("block") monotonic, align 41772 %c.add = add i32 %c.load, 11773 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("block") monotonic, align 41774 1775 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("block") monotonic, align 81776 %d.add = add i64 %d.load, 11777 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("block") monotonic, align 81778 1779 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("block") monotonic, align 41780 %e.add = fadd float %e.load, 1.1781 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("block") monotonic, align 41782 1783 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("block") monotonic, align 81784 %f.add = fadd double %f.load, 1.1785 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("block") monotonic, align 81786 1787 ret void1788}1789 1790; CHECK-LABEL: global_acq_rel_sys1791define void @global_acq_rel_sys(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1792; CHECK-LABEL: global_acq_rel_sys(1793; CHECK: {1794; CHECK-NEXT: .reg .b16 %rs<5>;1795; CHECK-NEXT: .reg .b32 %r<5>;1796; CHECK-NEXT: .reg .b64 %rd<10>;1797; CHECK-EMPTY:1798; CHECK-NEXT: // %bb.0:1799; CHECK-NEXT: ld.param.b64 %rd1, [global_acq_rel_sys_param_0];1800; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];1801; CHECK-NEXT: ld.param.b64 %rd2, [global_acq_rel_sys_param_1];1802; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1803; CHECK-NEXT: ld.param.b64 %rd3, [global_acq_rel_sys_param_2];1804; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;1805; CHECK-NEXT: ld.param.b64 %rd4, [global_acq_rel_sys_param_3];1806; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];1807; CHECK-NEXT: ld.param.b64 %rd5, [global_acq_rel_sys_param_4];1808; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1809; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;1810; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];1811; CHECK-NEXT: add.s32 %r2, %r1, 1;1812; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;1813; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];1814; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1815; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;1816; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];1817; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1818; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;1819; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];1820; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1821; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;1822; CHECK-NEXT: ret;1823 %a.load = load atomic i8, ptr addrspace(1) %a acquire, align 11824 %a.add = add i8 %a.load, 11825 store atomic i8 %a.add, ptr addrspace(1) %a release, align 11826 1827 %b.load = load atomic i16, ptr addrspace(1) %b acquire, align 21828 %b.add = add i16 %b.load, 11829 store atomic i16 %b.add, ptr addrspace(1) %b release, align 21830 1831 %c.load = load atomic i32, ptr addrspace(1) %c acquire, align 41832 %c.add = add i32 %c.load, 11833 store atomic i32 %c.add, ptr addrspace(1) %c release, align 41834 1835 %d.load = load atomic i64, ptr addrspace(1) %d acquire, align 81836 %d.add = add i64 %d.load, 11837 store atomic i64 %d.add, ptr addrspace(1) %d release, align 81838 1839 %e.load = load atomic float, ptr addrspace(1) %e acquire, align 41840 %e.add = fadd float %e.load, 1.1841 store atomic float %e.add, ptr addrspace(1) %e release, align 41842 1843 %f.load = load atomic double, ptr addrspace(1) %e acquire, align 81844 %f.add = fadd double %f.load, 1.1845 store atomic double %f.add, ptr addrspace(1) %e release, align 81846 1847 ret void1848}1849 1850; CHECK-LABEL: global_acq_rel_volatile_sys1851define void @global_acq_rel_volatile_sys(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1852; CHECK-LABEL: global_acq_rel_volatile_sys(1853; CHECK: {1854; CHECK-NEXT: .reg .b16 %rs<5>;1855; CHECK-NEXT: .reg .b32 %r<5>;1856; CHECK-NEXT: .reg .b64 %rd<10>;1857; CHECK-EMPTY:1858; CHECK-NEXT: // %bb.0:1859; CHECK-NEXT: ld.param.b64 %rd1, [global_acq_rel_volatile_sys_param_0];1860; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];1861; CHECK-NEXT: ld.param.b64 %rd2, [global_acq_rel_volatile_sys_param_1];1862; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1863; CHECK-NEXT: ld.param.b64 %rd3, [global_acq_rel_volatile_sys_param_2];1864; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;1865; CHECK-NEXT: ld.param.b64 %rd4, [global_acq_rel_volatile_sys_param_3];1866; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];1867; CHECK-NEXT: ld.param.b64 %rd5, [global_acq_rel_volatile_sys_param_4];1868; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1869; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;1870; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];1871; CHECK-NEXT: add.s32 %r2, %r1, 1;1872; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;1873; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];1874; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1875; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;1876; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];1877; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1878; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;1879; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];1880; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1881; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;1882; CHECK-NEXT: ret;1883 %a.load = load atomic volatile i8, ptr addrspace(1) %a acquire, align 11884 %a.add = add i8 %a.load, 11885 store atomic volatile i8 %a.add, ptr addrspace(1) %a release, align 11886 1887 %b.load = load atomic volatile i16, ptr addrspace(1) %b acquire, align 21888 %b.add = add i16 %b.load, 11889 store atomic volatile i16 %b.add, ptr addrspace(1) %b release, align 21890 1891 %c.load = load atomic volatile i32, ptr addrspace(1) %c acquire, align 41892 %c.add = add i32 %c.load, 11893 store atomic volatile i32 %c.add, ptr addrspace(1) %c release, align 41894 1895 %d.load = load atomic volatile i64, ptr addrspace(1) %d acquire, align 81896 %d.add = add i64 %d.load, 11897 store atomic volatile i64 %d.add, ptr addrspace(1) %d release, align 81898 1899 %e.load = load atomic volatile float, ptr addrspace(1) %e acquire, align 41900 %e.add = fadd float %e.load, 1.1901 store atomic volatile float %e.add, ptr addrspace(1) %e release, align 41902 1903 %f.load = load atomic volatile double, ptr addrspace(1) %e acquire, align 81904 %f.add = fadd double %f.load, 1.1905 store atomic volatile double %f.add, ptr addrspace(1) %e release, align 81906 1907 ret void1908}1909 1910; CHECK-LABEL: global_acq_rel_gpu1911define void @global_acq_rel_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1912; CHECK-LABEL: global_acq_rel_gpu(1913; CHECK: {1914; CHECK-NEXT: .reg .b16 %rs<5>;1915; CHECK-NEXT: .reg .b32 %r<5>;1916; CHECK-NEXT: .reg .b64 %rd<10>;1917; CHECK-EMPTY:1918; CHECK-NEXT: // %bb.0:1919; CHECK-NEXT: ld.param.b64 %rd1, [global_acq_rel_gpu_param_0];1920; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];1921; CHECK-NEXT: ld.param.b64 %rd2, [global_acq_rel_gpu_param_1];1922; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1923; CHECK-NEXT: ld.param.b64 %rd3, [global_acq_rel_gpu_param_2];1924; CHECK-NEXT: st.release.gpu.global.b8 [%rd1], %rs2;1925; CHECK-NEXT: ld.param.b64 %rd4, [global_acq_rel_gpu_param_3];1926; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs3, [%rd2];1927; CHECK-NEXT: ld.param.b64 %rd5, [global_acq_rel_gpu_param_4];1928; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1929; CHECK-NEXT: st.release.gpu.global.b16 [%rd2], %rs4;1930; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd3];1931; CHECK-NEXT: add.s32 %r2, %r1, 1;1932; CHECK-NEXT: st.release.gpu.global.b32 [%rd3], %r2;1933; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd6, [%rd4];1934; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1935; CHECK-NEXT: st.release.gpu.global.b64 [%rd4], %rd7;1936; CHECK-NEXT: ld.acquire.gpu.global.b32 %r3, [%rd5];1937; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1938; CHECK-NEXT: st.release.gpu.global.b32 [%rd5], %r4;1939; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd8, [%rd5];1940; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;1941; CHECK-NEXT: st.release.gpu.global.b64 [%rd5], %rd9;1942; CHECK-NEXT: ret;1943 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("device") acquire, align 11944 %a.add = add i8 %a.load, 11945 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("device") release, align 11946 1947 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("device") acquire, align 21948 %b.add = add i16 %b.load, 11949 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("device") release, align 21950 1951 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("device") acquire, align 41952 %c.add = add i32 %c.load, 11953 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("device") release, align 41954 1955 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("device") acquire, align 81956 %d.add = add i64 %d.load, 11957 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("device") release, align 81958 1959 %e.load = load atomic float, ptr addrspace(1) %e syncscope("device") acquire, align 41960 %e.add = fadd float %e.load, 1.1961 store atomic float %e.add, ptr addrspace(1) %e syncscope("device") release, align 41962 1963 %f.load = load atomic double, ptr addrspace(1) %e syncscope("device") acquire, align 81964 %f.add = fadd double %f.load, 1.1965 store atomic double %f.add, ptr addrspace(1) %e syncscope("device") release, align 81966 1967 ret void1968}1969 1970; CHECK-LABEL: global_acq_rel_volatile_gpu1971define void @global_acq_rel_volatile_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {1972; CHECK-LABEL: global_acq_rel_volatile_gpu(1973; CHECK: {1974; CHECK-NEXT: .reg .b16 %rs<5>;1975; CHECK-NEXT: .reg .b32 %r<5>;1976; CHECK-NEXT: .reg .b64 %rd<10>;1977; CHECK-EMPTY:1978; CHECK-NEXT: // %bb.0:1979; CHECK-NEXT: ld.param.b64 %rd1, [global_acq_rel_volatile_gpu_param_0];1980; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];1981; CHECK-NEXT: ld.param.b64 %rd2, [global_acq_rel_volatile_gpu_param_1];1982; CHECK-NEXT: add.s16 %rs2, %rs1, 1;1983; CHECK-NEXT: ld.param.b64 %rd3, [global_acq_rel_volatile_gpu_param_2];1984; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;1985; CHECK-NEXT: ld.param.b64 %rd4, [global_acq_rel_volatile_gpu_param_3];1986; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];1987; CHECK-NEXT: ld.param.b64 %rd5, [global_acq_rel_volatile_gpu_param_4];1988; CHECK-NEXT: add.s16 %rs4, %rs3, 1;1989; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;1990; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];1991; CHECK-NEXT: add.s32 %r2, %r1, 1;1992; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;1993; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];1994; CHECK-NEXT: add.s64 %rd7, %rd6, 1;1995; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;1996; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];1997; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;1998; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;1999; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];2000; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2001; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;2002; CHECK-NEXT: ret;2003 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("device") acquire, align 12004 %a.add = add i8 %a.load, 12005 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("device") release, align 12006 2007 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("device") acquire, align 22008 %b.add = add i16 %b.load, 12009 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("device") release, align 22010 2011 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("device") acquire, align 42012 %c.add = add i32 %c.load, 12013 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("device") release, align 42014 2015 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("device") acquire, align 82016 %d.add = add i64 %d.load, 12017 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("device") release, align 82018 2019 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("device") acquire, align 42020 %e.add = fadd float %e.load, 1.2021 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("device") release, align 42022 2023 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("device") acquire, align 82024 %f.add = fadd double %f.load, 1.2025 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("device") release, align 82026 2027 ret void2028}2029 2030; CHECK-LABEL: global_acq_rel_cta2031define void @global_acq_rel_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2032; CHECK-LABEL: global_acq_rel_cta(2033; CHECK: {2034; CHECK-NEXT: .reg .b16 %rs<5>;2035; CHECK-NEXT: .reg .b32 %r<5>;2036; CHECK-NEXT: .reg .b64 %rd<10>;2037; CHECK-EMPTY:2038; CHECK-NEXT: // %bb.0:2039; CHECK-NEXT: ld.param.b64 %rd1, [global_acq_rel_cta_param_0];2040; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];2041; CHECK-NEXT: ld.param.b64 %rd2, [global_acq_rel_cta_param_1];2042; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2043; CHECK-NEXT: ld.param.b64 %rd3, [global_acq_rel_cta_param_2];2044; CHECK-NEXT: st.release.cta.global.b8 [%rd1], %rs2;2045; CHECK-NEXT: ld.param.b64 %rd4, [global_acq_rel_cta_param_3];2046; CHECK-NEXT: ld.acquire.cta.global.b16 %rs3, [%rd2];2047; CHECK-NEXT: ld.param.b64 %rd5, [global_acq_rel_cta_param_4];2048; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2049; CHECK-NEXT: st.release.cta.global.b16 [%rd2], %rs4;2050; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd3];2051; CHECK-NEXT: add.s32 %r2, %r1, 1;2052; CHECK-NEXT: st.release.cta.global.b32 [%rd3], %r2;2053; CHECK-NEXT: ld.acquire.cta.global.b64 %rd6, [%rd4];2054; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2055; CHECK-NEXT: st.release.cta.global.b64 [%rd4], %rd7;2056; CHECK-NEXT: ld.acquire.cta.global.b32 %r3, [%rd5];2057; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2058; CHECK-NEXT: st.release.cta.global.b32 [%rd5], %r4;2059; CHECK-NEXT: ld.acquire.cta.global.b64 %rd8, [%rd5];2060; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2061; CHECK-NEXT: st.release.cta.global.b64 [%rd5], %rd9;2062; CHECK-NEXT: ret;2063 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("block") acquire, align 12064 %a.add = add i8 %a.load, 12065 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("block") release, align 12066 2067 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("block") acquire, align 22068 %b.add = add i16 %b.load, 12069 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("block") release, align 22070 2071 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("block") acquire, align 42072 %c.add = add i32 %c.load, 12073 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("block") release, align 42074 2075 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("block") acquire, align 82076 %d.add = add i64 %d.load, 12077 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("block") release, align 82078 2079 %e.load = load atomic float, ptr addrspace(1) %e syncscope("block") acquire, align 42080 %e.add = fadd float %e.load, 1.2081 store atomic float %e.add, ptr addrspace(1) %e syncscope("block") release, align 42082 2083 %f.load = load atomic double, ptr addrspace(1) %e syncscope("block") acquire, align 82084 %f.add = fadd double %f.load, 1.2085 store atomic double %f.add, ptr addrspace(1) %e syncscope("block") release, align 82086 2087 ret void2088}2089 2090; CHECK-LABEL: global_acq_rel_volatile_cta2091define void @global_acq_rel_volatile_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2092; CHECK-LABEL: global_acq_rel_volatile_cta(2093; CHECK: {2094; CHECK-NEXT: .reg .b16 %rs<5>;2095; CHECK-NEXT: .reg .b32 %r<5>;2096; CHECK-NEXT: .reg .b64 %rd<10>;2097; CHECK-EMPTY:2098; CHECK-NEXT: // %bb.0:2099; CHECK-NEXT: ld.param.b64 %rd1, [global_acq_rel_volatile_cta_param_0];2100; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];2101; CHECK-NEXT: ld.param.b64 %rd2, [global_acq_rel_volatile_cta_param_1];2102; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2103; CHECK-NEXT: ld.param.b64 %rd3, [global_acq_rel_volatile_cta_param_2];2104; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;2105; CHECK-NEXT: ld.param.b64 %rd4, [global_acq_rel_volatile_cta_param_3];2106; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];2107; CHECK-NEXT: ld.param.b64 %rd5, [global_acq_rel_volatile_cta_param_4];2108; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2109; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;2110; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];2111; CHECK-NEXT: add.s32 %r2, %r1, 1;2112; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;2113; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];2114; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2115; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;2116; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];2117; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2118; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;2119; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];2120; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2121; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;2122; CHECK-NEXT: ret;2123 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("block") acquire, align 12124 %a.add = add i8 %a.load, 12125 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("block") release, align 12126 2127 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("block") acquire, align 22128 %b.add = add i16 %b.load, 12129 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("block") release, align 22130 2131 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("block") acquire, align 42132 %c.add = add i32 %c.load, 12133 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("block") release, align 42134 2135 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("block") acquire, align 82136 %d.add = add i64 %d.load, 12137 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("block") release, align 82138 2139 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("block") acquire, align 42140 %e.add = fadd float %e.load, 1.2141 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("block") release, align 42142 2143 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("block") acquire, align 82144 %f.add = fadd double %f.load, 1.2145 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("block") release, align 82146 2147 ret void2148}2149 2150; CHECK-LABEL: global_seq_cst_sys2151define void @global_seq_cst_sys(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2152; CHECK-LABEL: global_seq_cst_sys(2153; CHECK: {2154; CHECK-NEXT: .reg .b16 %rs<5>;2155; CHECK-NEXT: .reg .b32 %r<5>;2156; CHECK-NEXT: .reg .b64 %rd<10>;2157; CHECK-EMPTY:2158; CHECK-NEXT: // %bb.0:2159; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_sys_param_0];2160; CHECK-NEXT: fence.sc.sys;2161; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];2162; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_sys_param_1];2163; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2164; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_sys_param_2];2165; CHECK-NEXT: fence.sc.sys;2166; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;2167; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_sys_param_3];2168; CHECK-NEXT: fence.sc.sys;2169; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];2170; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_sys_param_4];2171; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2172; CHECK-NEXT: fence.sc.sys;2173; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;2174; CHECK-NEXT: fence.sc.sys;2175; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];2176; CHECK-NEXT: add.s32 %r2, %r1, 1;2177; CHECK-NEXT: fence.sc.sys;2178; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;2179; CHECK-NEXT: fence.sc.sys;2180; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];2181; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2182; CHECK-NEXT: fence.sc.sys;2183; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;2184; CHECK-NEXT: fence.sc.sys;2185; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];2186; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2187; CHECK-NEXT: fence.sc.sys;2188; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;2189; CHECK-NEXT: fence.sc.sys;2190; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];2191; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2192; CHECK-NEXT: fence.sc.sys;2193; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;2194; CHECK-NEXT: ret;2195 %a.load = load atomic i8, ptr addrspace(1) %a seq_cst, align 12196 %a.add = add i8 %a.load, 12197 store atomic i8 %a.add, ptr addrspace(1) %a seq_cst, align 12198 2199 %b.load = load atomic i16, ptr addrspace(1) %b seq_cst, align 22200 %b.add = add i16 %b.load, 12201 store atomic i16 %b.add, ptr addrspace(1) %b seq_cst, align 22202 2203 %c.load = load atomic i32, ptr addrspace(1) %c seq_cst, align 42204 %c.add = add i32 %c.load, 12205 store atomic i32 %c.add, ptr addrspace(1) %c seq_cst, align 42206 2207 %d.load = load atomic i64, ptr addrspace(1) %d seq_cst, align 82208 %d.add = add i64 %d.load, 12209 store atomic i64 %d.add, ptr addrspace(1) %d seq_cst, align 82210 2211 %e.load = load atomic float, ptr addrspace(1) %e seq_cst, align 42212 %e.add = fadd float %e.load, 1.2213 store atomic float %e.add, ptr addrspace(1) %e seq_cst, align 42214 2215 %f.load = load atomic double, ptr addrspace(1) %e seq_cst, align 82216 %f.add = fadd double %f.load, 1.2217 store atomic double %f.add, ptr addrspace(1) %e seq_cst, align 82218 2219 ret void2220}2221 2222; CHECK-LABEL: global_seq_cst_volatile_sys2223define void @global_seq_cst_volatile_sys(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2224; CHECK-LABEL: global_seq_cst_volatile_sys(2225; CHECK: {2226; CHECK-NEXT: .reg .b16 %rs<5>;2227; CHECK-NEXT: .reg .b32 %r<5>;2228; CHECK-NEXT: .reg .b64 %rd<10>;2229; CHECK-EMPTY:2230; CHECK-NEXT: // %bb.0:2231; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_volatile_sys_param_0];2232; CHECK-NEXT: fence.sc.sys;2233; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];2234; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_volatile_sys_param_1];2235; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2236; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_volatile_sys_param_2];2237; CHECK-NEXT: fence.sc.sys;2238; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;2239; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_volatile_sys_param_3];2240; CHECK-NEXT: fence.sc.sys;2241; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];2242; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_volatile_sys_param_4];2243; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2244; CHECK-NEXT: fence.sc.sys;2245; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;2246; CHECK-NEXT: fence.sc.sys;2247; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];2248; CHECK-NEXT: add.s32 %r2, %r1, 1;2249; CHECK-NEXT: fence.sc.sys;2250; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;2251; CHECK-NEXT: fence.sc.sys;2252; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];2253; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2254; CHECK-NEXT: fence.sc.sys;2255; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;2256; CHECK-NEXT: fence.sc.sys;2257; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];2258; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2259; CHECK-NEXT: fence.sc.sys;2260; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;2261; CHECK-NEXT: fence.sc.sys;2262; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];2263; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2264; CHECK-NEXT: fence.sc.sys;2265; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;2266; CHECK-NEXT: ret;2267 %a.load = load atomic volatile i8, ptr addrspace(1) %a seq_cst, align 12268 %a.add = add i8 %a.load, 12269 store atomic volatile i8 %a.add, ptr addrspace(1) %a seq_cst, align 12270 2271 %b.load = load atomic volatile i16, ptr addrspace(1) %b seq_cst, align 22272 %b.add = add i16 %b.load, 12273 store atomic volatile i16 %b.add, ptr addrspace(1) %b seq_cst, align 22274 2275 %c.load = load atomic volatile i32, ptr addrspace(1) %c seq_cst, align 42276 %c.add = add i32 %c.load, 12277 store atomic volatile i32 %c.add, ptr addrspace(1) %c seq_cst, align 42278 2279 %d.load = load atomic volatile i64, ptr addrspace(1) %d seq_cst, align 82280 %d.add = add i64 %d.load, 12281 store atomic volatile i64 %d.add, ptr addrspace(1) %d seq_cst, align 82282 2283 %e.load = load atomic volatile float, ptr addrspace(1) %e seq_cst, align 42284 %e.add = fadd float %e.load, 1.2285 store atomic volatile float %e.add, ptr addrspace(1) %e seq_cst, align 42286 2287 %f.load = load atomic volatile double, ptr addrspace(1) %e seq_cst, align 82288 %f.add = fadd double %f.load, 1.2289 store atomic volatile double %f.add, ptr addrspace(1) %e seq_cst, align 82290 2291 ret void2292}2293 2294; CHECK-LABEL: global_seq_cst_gpu2295define void @global_seq_cst_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2296; CHECK-LABEL: global_seq_cst_gpu(2297; CHECK: {2298; CHECK-NEXT: .reg .b16 %rs<5>;2299; CHECK-NEXT: .reg .b32 %r<5>;2300; CHECK-NEXT: .reg .b64 %rd<10>;2301; CHECK-EMPTY:2302; CHECK-NEXT: // %bb.0:2303; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_gpu_param_0];2304; CHECK-NEXT: fence.sc.gpu;2305; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];2306; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_gpu_param_1];2307; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2308; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_gpu_param_2];2309; CHECK-NEXT: fence.sc.gpu;2310; CHECK-NEXT: st.release.gpu.global.b8 [%rd1], %rs2;2311; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_gpu_param_3];2312; CHECK-NEXT: fence.sc.gpu;2313; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs3, [%rd2];2314; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_gpu_param_4];2315; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2316; CHECK-NEXT: fence.sc.gpu;2317; CHECK-NEXT: st.release.gpu.global.b16 [%rd2], %rs4;2318; CHECK-NEXT: fence.sc.gpu;2319; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd3];2320; CHECK-NEXT: add.s32 %r2, %r1, 1;2321; CHECK-NEXT: fence.sc.gpu;2322; CHECK-NEXT: st.release.gpu.global.b32 [%rd3], %r2;2323; CHECK-NEXT: fence.sc.gpu;2324; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd6, [%rd4];2325; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2326; CHECK-NEXT: fence.sc.gpu;2327; CHECK-NEXT: st.release.gpu.global.b64 [%rd4], %rd7;2328; CHECK-NEXT: fence.sc.gpu;2329; CHECK-NEXT: ld.acquire.gpu.global.b32 %r3, [%rd5];2330; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2331; CHECK-NEXT: fence.sc.gpu;2332; CHECK-NEXT: st.release.gpu.global.b32 [%rd5], %r4;2333; CHECK-NEXT: fence.sc.gpu;2334; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd8, [%rd5];2335; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2336; CHECK-NEXT: fence.sc.gpu;2337; CHECK-NEXT: st.release.gpu.global.b64 [%rd5], %rd9;2338; CHECK-NEXT: ret;2339 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("device") seq_cst, align 12340 %a.add = add i8 %a.load, 12341 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("device") seq_cst, align 12342 2343 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("device") seq_cst, align 22344 %b.add = add i16 %b.load, 12345 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("device") seq_cst, align 22346 2347 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("device") seq_cst, align 42348 %c.add = add i32 %c.load, 12349 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("device") seq_cst, align 42350 2351 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("device") seq_cst, align 82352 %d.add = add i64 %d.load, 12353 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("device") seq_cst, align 82354 2355 %e.load = load atomic float, ptr addrspace(1) %e syncscope("device") seq_cst, align 42356 %e.add = fadd float %e.load, 1.2357 store atomic float %e.add, ptr addrspace(1) %e syncscope("device") seq_cst, align 42358 2359 %f.load = load atomic double, ptr addrspace(1) %e syncscope("device") seq_cst, align 82360 %f.add = fadd double %f.load, 1.2361 store atomic double %f.add, ptr addrspace(1) %e syncscope("device") seq_cst, align 82362 2363 ret void2364}2365 2366; CHECK-LABEL: global_seq_cst_volatile_gpu2367define void @global_seq_cst_volatile_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2368; CHECK-LABEL: global_seq_cst_volatile_gpu(2369; CHECK: {2370; CHECK-NEXT: .reg .b16 %rs<5>;2371; CHECK-NEXT: .reg .b32 %r<5>;2372; CHECK-NEXT: .reg .b64 %rd<10>;2373; CHECK-EMPTY:2374; CHECK-NEXT: // %bb.0:2375; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_volatile_gpu_param_0];2376; CHECK-NEXT: fence.sc.sys;2377; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];2378; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_volatile_gpu_param_1];2379; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2380; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_volatile_gpu_param_2];2381; CHECK-NEXT: fence.sc.sys;2382; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;2383; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_volatile_gpu_param_3];2384; CHECK-NEXT: fence.sc.sys;2385; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];2386; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_volatile_gpu_param_4];2387; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2388; CHECK-NEXT: fence.sc.sys;2389; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;2390; CHECK-NEXT: fence.sc.sys;2391; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];2392; CHECK-NEXT: add.s32 %r2, %r1, 1;2393; CHECK-NEXT: fence.sc.sys;2394; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;2395; CHECK-NEXT: fence.sc.sys;2396; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];2397; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2398; CHECK-NEXT: fence.sc.sys;2399; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;2400; CHECK-NEXT: fence.sc.sys;2401; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];2402; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2403; CHECK-NEXT: fence.sc.sys;2404; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;2405; CHECK-NEXT: fence.sc.sys;2406; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];2407; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2408; CHECK-NEXT: fence.sc.sys;2409; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;2410; CHECK-NEXT: ret;2411 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("device") seq_cst, align 12412 %a.add = add i8 %a.load, 12413 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("device") seq_cst, align 12414 2415 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("device") seq_cst, align 22416 %b.add = add i16 %b.load, 12417 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("device") seq_cst, align 22418 2419 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("device") seq_cst, align 42420 %c.add = add i32 %c.load, 12421 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("device") seq_cst, align 42422 2423 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("device") seq_cst, align 82424 %d.add = add i64 %d.load, 12425 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("device") seq_cst, align 82426 2427 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("device") seq_cst, align 42428 %e.add = fadd float %e.load, 1.2429 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("device") seq_cst, align 42430 2431 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("device") seq_cst, align 82432 %f.add = fadd double %f.load, 1.2433 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("device") seq_cst, align 82434 2435 ret void2436}2437 2438; CHECK-LABEL: global_seq_cst_cta2439define void @global_seq_cst_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2440; CHECK-LABEL: global_seq_cst_cta(2441; CHECK: {2442; CHECK-NEXT: .reg .b16 %rs<5>;2443; CHECK-NEXT: .reg .b32 %r<5>;2444; CHECK-NEXT: .reg .b64 %rd<10>;2445; CHECK-EMPTY:2446; CHECK-NEXT: // %bb.0:2447; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_cta_param_0];2448; CHECK-NEXT: fence.sc.cta;2449; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];2450; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_cta_param_1];2451; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2452; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_cta_param_2];2453; CHECK-NEXT: fence.sc.cta;2454; CHECK-NEXT: st.release.cta.global.b8 [%rd1], %rs2;2455; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_cta_param_3];2456; CHECK-NEXT: fence.sc.cta;2457; CHECK-NEXT: ld.acquire.cta.global.b16 %rs3, [%rd2];2458; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_cta_param_4];2459; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2460; CHECK-NEXT: fence.sc.cta;2461; CHECK-NEXT: st.release.cta.global.b16 [%rd2], %rs4;2462; CHECK-NEXT: fence.sc.cta;2463; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd3];2464; CHECK-NEXT: add.s32 %r2, %r1, 1;2465; CHECK-NEXT: fence.sc.cta;2466; CHECK-NEXT: st.release.cta.global.b32 [%rd3], %r2;2467; CHECK-NEXT: fence.sc.cta;2468; CHECK-NEXT: ld.acquire.cta.global.b64 %rd6, [%rd4];2469; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2470; CHECK-NEXT: fence.sc.cta;2471; CHECK-NEXT: st.release.cta.global.b64 [%rd4], %rd7;2472; CHECK-NEXT: fence.sc.cta;2473; CHECK-NEXT: ld.acquire.cta.global.b32 %r3, [%rd5];2474; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2475; CHECK-NEXT: fence.sc.cta;2476; CHECK-NEXT: st.release.cta.global.b32 [%rd5], %r4;2477; CHECK-NEXT: fence.sc.cta;2478; CHECK-NEXT: ld.acquire.cta.global.b64 %rd8, [%rd5];2479; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2480; CHECK-NEXT: fence.sc.cta;2481; CHECK-NEXT: st.release.cta.global.b64 [%rd5], %rd9;2482; CHECK-NEXT: ret;2483 %a.load = load atomic i8, ptr addrspace(1) %a syncscope("block") seq_cst, align 12484 %a.add = add i8 %a.load, 12485 store atomic i8 %a.add, ptr addrspace(1) %a syncscope("block") seq_cst, align 12486 2487 %b.load = load atomic i16, ptr addrspace(1) %b syncscope("block") seq_cst, align 22488 %b.add = add i16 %b.load, 12489 store atomic i16 %b.add, ptr addrspace(1) %b syncscope("block") seq_cst, align 22490 2491 %c.load = load atomic i32, ptr addrspace(1) %c syncscope("block") seq_cst, align 42492 %c.add = add i32 %c.load, 12493 store atomic i32 %c.add, ptr addrspace(1) %c syncscope("block") seq_cst, align 42494 2495 %d.load = load atomic i64, ptr addrspace(1) %d syncscope("block") seq_cst, align 82496 %d.add = add i64 %d.load, 12497 store atomic i64 %d.add, ptr addrspace(1) %d syncscope("block") seq_cst, align 82498 2499 %e.load = load atomic float, ptr addrspace(1) %e syncscope("block") seq_cst, align 42500 %e.add = fadd float %e.load, 1.2501 store atomic float %e.add, ptr addrspace(1) %e syncscope("block") seq_cst, align 42502 2503 %f.load = load atomic double, ptr addrspace(1) %e syncscope("block") seq_cst, align 82504 %f.add = fadd double %f.load, 1.2505 store atomic double %f.add, ptr addrspace(1) %e syncscope("block") seq_cst, align 82506 2507 ret void2508}2509 2510; CHECK-LABEL: global_seq_cst_volatile_cta2511define void @global_seq_cst_volatile_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %d, ptr addrspace(1) %e) local_unnamed_addr {2512; CHECK-LABEL: global_seq_cst_volatile_cta(2513; CHECK: {2514; CHECK-NEXT: .reg .b16 %rs<5>;2515; CHECK-NEXT: .reg .b32 %r<5>;2516; CHECK-NEXT: .reg .b64 %rd<10>;2517; CHECK-EMPTY:2518; CHECK-NEXT: // %bb.0:2519; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_volatile_cta_param_0];2520; CHECK-NEXT: fence.sc.sys;2521; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];2522; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_volatile_cta_param_1];2523; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2524; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_volatile_cta_param_2];2525; CHECK-NEXT: fence.sc.sys;2526; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;2527; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_volatile_cta_param_3];2528; CHECK-NEXT: fence.sc.sys;2529; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];2530; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_volatile_cta_param_4];2531; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2532; CHECK-NEXT: fence.sc.sys;2533; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;2534; CHECK-NEXT: fence.sc.sys;2535; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];2536; CHECK-NEXT: add.s32 %r2, %r1, 1;2537; CHECK-NEXT: fence.sc.sys;2538; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;2539; CHECK-NEXT: fence.sc.sys;2540; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];2541; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2542; CHECK-NEXT: fence.sc.sys;2543; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;2544; CHECK-NEXT: fence.sc.sys;2545; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];2546; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2547; CHECK-NEXT: fence.sc.sys;2548; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;2549; CHECK-NEXT: fence.sc.sys;2550; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];2551; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2552; CHECK-NEXT: fence.sc.sys;2553; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;2554; CHECK-NEXT: ret;2555 %a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("block") seq_cst, align 12556 %a.add = add i8 %a.load, 12557 store atomic volatile i8 %a.add, ptr addrspace(1) %a syncscope("block") seq_cst, align 12558 2559 %b.load = load atomic volatile i16, ptr addrspace(1) %b syncscope("block") seq_cst, align 22560 %b.add = add i16 %b.load, 12561 store atomic volatile i16 %b.add, ptr addrspace(1) %b syncscope("block") seq_cst, align 22562 2563 %c.load = load atomic volatile i32, ptr addrspace(1) %c syncscope("block") seq_cst, align 42564 %c.add = add i32 %c.load, 12565 store atomic volatile i32 %c.add, ptr addrspace(1) %c syncscope("block") seq_cst, align 42566 2567 %d.load = load atomic volatile i64, ptr addrspace(1) %d syncscope("block") seq_cst, align 82568 %d.add = add i64 %d.load, 12569 store atomic volatile i64 %d.add, ptr addrspace(1) %d syncscope("block") seq_cst, align 82570 2571 %e.load = load atomic volatile float, ptr addrspace(1) %e syncscope("block") seq_cst, align 42572 %e.add = fadd float %e.load, 1.2573 store atomic volatile float %e.add, ptr addrspace(1) %e syncscope("block") seq_cst, align 42574 2575 %f.load = load atomic volatile double, ptr addrspace(1) %e syncscope("block") seq_cst, align 82576 %f.add = fadd double %f.load, 1.2577 store atomic volatile double %f.add, ptr addrspace(1) %e syncscope("block") seq_cst, align 82578 2579 ret void2580}2581 2582;; shared statespace2583 2584; CHECK-LABEL: shared_unordered_gpu2585define void @shared_unordered_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {2586; CHECK-LABEL: shared_unordered_gpu(2587; CHECK: {2588; CHECK-NEXT: .reg .b16 %rs<5>;2589; CHECK-NEXT: .reg .b32 %r<5>;2590; CHECK-NEXT: .reg .b64 %rd<10>;2591; CHECK-EMPTY:2592; CHECK-NEXT: // %bb.0:2593; CHECK-NEXT: ld.param.b64 %rd1, [shared_unordered_gpu_param_0];2594; CHECK-NEXT: ld.relaxed.gpu.shared.b8 %rs1, [%rd1];2595; CHECK-NEXT: ld.param.b64 %rd2, [shared_unordered_gpu_param_1];2596; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2597; CHECK-NEXT: ld.param.b64 %rd3, [shared_unordered_gpu_param_2];2598; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs2;2599; CHECK-NEXT: ld.param.b64 %rd4, [shared_unordered_gpu_param_3];2600; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs3, [%rd2];2601; CHECK-NEXT: ld.param.b64 %rd5, [shared_unordered_gpu_param_4];2602; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2603; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd2], %rs4;2604; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd3];2605; CHECK-NEXT: add.s32 %r2, %r1, 1;2606; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd3], %r2;2607; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd6, [%rd4];2608; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2609; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd4], %rd7;2610; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r3, [%rd5];2611; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2612; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd5], %r4;2613; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd8, [%rd5];2614; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2615; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd5], %rd9;2616; CHECK-NEXT: ret;2617 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("device") unordered, align 12618 %a.add = add i8 %a.load, 12619 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("device") unordered, align 12620 2621 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("device") unordered, align 22622 %b.add = add i16 %b.load, 12623 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("device") unordered, align 22624 2625 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("device") unordered, align 42626 %c.add = add i32 %c.load, 12627 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("device") unordered, align 42628 2629 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("device") unordered, align 82630 %d.add = add i64 %d.load, 12631 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("device") unordered, align 82632 2633 %e.load = load atomic float, ptr addrspace(3) %e syncscope("device") unordered, align 42634 %e.add = fadd float %e.load, 1.2635 store atomic float %e.add, ptr addrspace(3) %e syncscope("device") unordered, align 42636 2637 %f.load = load atomic double, ptr addrspace(3) %e syncscope("device") unordered, align 82638 %f.add = fadd double %f.load, 1.2639 store atomic double %f.add, ptr addrspace(3) %e syncscope("device") unordered, align 82640 2641 ret void2642}2643 2644; CHECK-LABEL: shared_unordered_volatile_gpu2645define void @shared_unordered_volatile_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {2646; CHECK-LABEL: shared_unordered_volatile_gpu(2647; CHECK: {2648; CHECK-NEXT: .reg .b16 %rs<5>;2649; CHECK-NEXT: .reg .b32 %r<5>;2650; CHECK-NEXT: .reg .b64 %rd<10>;2651; CHECK-EMPTY:2652; CHECK-NEXT: // %bb.0:2653; CHECK-NEXT: ld.param.b64 %rd1, [shared_unordered_volatile_gpu_param_0];2654; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];2655; CHECK-NEXT: ld.param.b64 %rd2, [shared_unordered_volatile_gpu_param_1];2656; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2657; CHECK-NEXT: ld.param.b64 %rd3, [shared_unordered_volatile_gpu_param_2];2658; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;2659; CHECK-NEXT: ld.param.b64 %rd4, [shared_unordered_volatile_gpu_param_3];2660; CHECK-NEXT: ld.volatile.shared.b16 %rs3, [%rd2];2661; CHECK-NEXT: ld.param.b64 %rd5, [shared_unordered_volatile_gpu_param_4];2662; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2663; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;2664; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd3];2665; CHECK-NEXT: add.s32 %r2, %r1, 1;2666; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;2667; CHECK-NEXT: ld.volatile.shared.b64 %rd6, [%rd4];2668; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2669; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;2670; CHECK-NEXT: ld.volatile.shared.b32 %r3, [%rd5];2671; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2672; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;2673; CHECK-NEXT: ld.volatile.shared.b64 %rd8, [%rd5];2674; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2675; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;2676; CHECK-NEXT: ret;2677 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("device") unordered, align 12678 %a.add = add i8 %a.load, 12679 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("device") unordered, align 12680 2681 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("device") unordered, align 22682 %b.add = add i16 %b.load, 12683 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("device") unordered, align 22684 2685 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("device") unordered, align 42686 %c.add = add i32 %c.load, 12687 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("device") unordered, align 42688 2689 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("device") unordered, align 82690 %d.add = add i64 %d.load, 12691 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("device") unordered, align 82692 2693 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("device") unordered, align 42694 %e.add = fadd float %e.load, 1.2695 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("device") unordered, align 42696 2697 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("device") unordered, align 82698 %f.add = fadd double %f.load, 1.2699 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("device") unordered, align 82700 2701 ret void2702}2703 2704; CHECK-LABEL: shared_unordered_cta2705define void @shared_unordered_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {2706; CHECK-LABEL: shared_unordered_cta(2707; CHECK: {2708; CHECK-NEXT: .reg .b16 %rs<5>;2709; CHECK-NEXT: .reg .b32 %r<5>;2710; CHECK-NEXT: .reg .b64 %rd<10>;2711; CHECK-EMPTY:2712; CHECK-NEXT: // %bb.0:2713; CHECK-NEXT: ld.param.b64 %rd1, [shared_unordered_cta_param_0];2714; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];2715; CHECK-NEXT: ld.param.b64 %rd2, [shared_unordered_cta_param_1];2716; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2717; CHECK-NEXT: ld.param.b64 %rd3, [shared_unordered_cta_param_2];2718; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs2;2719; CHECK-NEXT: ld.param.b64 %rd4, [shared_unordered_cta_param_3];2720; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs3, [%rd2];2721; CHECK-NEXT: ld.param.b64 %rd5, [shared_unordered_cta_param_4];2722; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2723; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd2], %rs4;2724; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd3];2725; CHECK-NEXT: add.s32 %r2, %r1, 1;2726; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd3], %r2;2727; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd6, [%rd4];2728; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2729; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd4], %rd7;2730; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r3, [%rd5];2731; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2732; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd5], %r4;2733; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd8, [%rd5];2734; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2735; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd5], %rd9;2736; CHECK-NEXT: ret;2737 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("block") unordered, align 12738 %a.add = add i8 %a.load, 12739 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("block") unordered, align 12740 2741 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("block") unordered, align 22742 %b.add = add i16 %b.load, 12743 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("block") unordered, align 22744 2745 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("block") unordered, align 42746 %c.add = add i32 %c.load, 12747 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("block") unordered, align 42748 2749 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("block") unordered, align 82750 %d.add = add i64 %d.load, 12751 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("block") unordered, align 82752 2753 %e.load = load atomic float, ptr addrspace(3) %e syncscope("block") unordered, align 42754 %e.add = fadd float %e.load, 1.2755 store atomic float %e.add, ptr addrspace(3) %e syncscope("block") unordered, align 42756 2757 %f.load = load atomic double, ptr addrspace(3) %e syncscope("block") unordered, align 82758 %f.add = fadd double %f.load, 1.2759 store atomic double %f.add, ptr addrspace(3) %e syncscope("block") unordered, align 82760 2761 ret void2762}2763 2764; CHECK-LABEL: shared_unordered_volatile_cta2765define void @shared_unordered_volatile_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {2766; CHECK-LABEL: shared_unordered_volatile_cta(2767; CHECK: {2768; CHECK-NEXT: .reg .b16 %rs<5>;2769; CHECK-NEXT: .reg .b32 %r<5>;2770; CHECK-NEXT: .reg .b64 %rd<10>;2771; CHECK-EMPTY:2772; CHECK-NEXT: // %bb.0:2773; CHECK-NEXT: ld.param.b64 %rd1, [shared_unordered_volatile_cta_param_0];2774; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];2775; CHECK-NEXT: ld.param.b64 %rd2, [shared_unordered_volatile_cta_param_1];2776; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2777; CHECK-NEXT: ld.param.b64 %rd3, [shared_unordered_volatile_cta_param_2];2778; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;2779; CHECK-NEXT: ld.param.b64 %rd4, [shared_unordered_volatile_cta_param_3];2780; CHECK-NEXT: ld.volatile.shared.b16 %rs3, [%rd2];2781; CHECK-NEXT: ld.param.b64 %rd5, [shared_unordered_volatile_cta_param_4];2782; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2783; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;2784; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd3];2785; CHECK-NEXT: add.s32 %r2, %r1, 1;2786; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;2787; CHECK-NEXT: ld.volatile.shared.b64 %rd6, [%rd4];2788; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2789; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;2790; CHECK-NEXT: ld.volatile.shared.b32 %r3, [%rd5];2791; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2792; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;2793; CHECK-NEXT: ld.volatile.shared.b64 %rd8, [%rd5];2794; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2795; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;2796; CHECK-NEXT: ret;2797 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("block") unordered, align 12798 %a.add = add i8 %a.load, 12799 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("block") unordered, align 12800 2801 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("block") unordered, align 22802 %b.add = add i16 %b.load, 12803 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("block") unordered, align 22804 2805 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("block") unordered, align 42806 %c.add = add i32 %c.load, 12807 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("block") unordered, align 42808 2809 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("block") unordered, align 82810 %d.add = add i64 %d.load, 12811 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("block") unordered, align 82812 2813 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("block") unordered, align 42814 %e.add = fadd float %e.load, 1.2815 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("block") unordered, align 42816 2817 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("block") unordered, align 82818 %f.add = fadd double %f.load, 1.2819 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("block") unordered, align 82820 2821 ret void2822}2823 2824; CHECK-LABEL: shared_monotonic_gpu2825define void @shared_monotonic_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {2826; CHECK-LABEL: shared_monotonic_gpu(2827; CHECK: {2828; CHECK-NEXT: .reg .b16 %rs<5>;2829; CHECK-NEXT: .reg .b32 %r<5>;2830; CHECK-NEXT: .reg .b64 %rd<10>;2831; CHECK-EMPTY:2832; CHECK-NEXT: // %bb.0:2833; CHECK-NEXT: ld.param.b64 %rd1, [shared_monotonic_gpu_param_0];2834; CHECK-NEXT: ld.relaxed.gpu.shared.b8 %rs1, [%rd1];2835; CHECK-NEXT: ld.param.b64 %rd2, [shared_monotonic_gpu_param_1];2836; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2837; CHECK-NEXT: ld.param.b64 %rd3, [shared_monotonic_gpu_param_2];2838; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs2;2839; CHECK-NEXT: ld.param.b64 %rd4, [shared_monotonic_gpu_param_3];2840; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs3, [%rd2];2841; CHECK-NEXT: ld.param.b64 %rd5, [shared_monotonic_gpu_param_4];2842; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2843; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd2], %rs4;2844; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd3];2845; CHECK-NEXT: add.s32 %r2, %r1, 1;2846; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd3], %r2;2847; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd6, [%rd4];2848; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2849; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd4], %rd7;2850; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r3, [%rd5];2851; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2852; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd5], %r4;2853; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd8, [%rd5];2854; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2855; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd5], %rd9;2856; CHECK-NEXT: ret;2857 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("device") monotonic, align 12858 %a.add = add i8 %a.load, 12859 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("device") monotonic, align 12860 2861 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("device") monotonic, align 22862 %b.add = add i16 %b.load, 12863 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("device") monotonic, align 22864 2865 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("device") monotonic, align 42866 %c.add = add i32 %c.load, 12867 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("device") monotonic, align 42868 2869 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("device") monotonic, align 82870 %d.add = add i64 %d.load, 12871 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("device") monotonic, align 82872 2873 %e.load = load atomic float, ptr addrspace(3) %e syncscope("device") monotonic, align 42874 %e.add = fadd float %e.load, 1.2875 store atomic float %e.add, ptr addrspace(3) %e syncscope("device") monotonic, align 42876 2877 %f.load = load atomic double, ptr addrspace(3) %e syncscope("device") monotonic, align 82878 %f.add = fadd double %f.load, 1.2879 store atomic double %f.add, ptr addrspace(3) %e syncscope("device") monotonic, align 82880 2881 ret void2882}2883 2884; CHECK-LABEL: shared_monotonic_volatile_gpu2885define void @shared_monotonic_volatile_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {2886; CHECK-LABEL: shared_monotonic_volatile_gpu(2887; CHECK: {2888; CHECK-NEXT: .reg .b16 %rs<5>;2889; CHECK-NEXT: .reg .b32 %r<5>;2890; CHECK-NEXT: .reg .b64 %rd<10>;2891; CHECK-EMPTY:2892; CHECK-NEXT: // %bb.0:2893; CHECK-NEXT: ld.param.b64 %rd1, [shared_monotonic_volatile_gpu_param_0];2894; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];2895; CHECK-NEXT: ld.param.b64 %rd2, [shared_monotonic_volatile_gpu_param_1];2896; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2897; CHECK-NEXT: ld.param.b64 %rd3, [shared_monotonic_volatile_gpu_param_2];2898; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;2899; CHECK-NEXT: ld.param.b64 %rd4, [shared_monotonic_volatile_gpu_param_3];2900; CHECK-NEXT: ld.volatile.shared.b16 %rs3, [%rd2];2901; CHECK-NEXT: ld.param.b64 %rd5, [shared_monotonic_volatile_gpu_param_4];2902; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2903; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;2904; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd3];2905; CHECK-NEXT: add.s32 %r2, %r1, 1;2906; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;2907; CHECK-NEXT: ld.volatile.shared.b64 %rd6, [%rd4];2908; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2909; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;2910; CHECK-NEXT: ld.volatile.shared.b32 %r3, [%rd5];2911; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2912; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;2913; CHECK-NEXT: ld.volatile.shared.b64 %rd8, [%rd5];2914; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2915; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;2916; CHECK-NEXT: ret;2917 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("device") monotonic, align 12918 %a.add = add i8 %a.load, 12919 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("device") monotonic, align 12920 2921 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("device") monotonic, align 22922 %b.add = add i16 %b.load, 12923 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("device") monotonic, align 22924 2925 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("device") monotonic, align 42926 %c.add = add i32 %c.load, 12927 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("device") monotonic, align 42928 2929 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("device") monotonic, align 82930 %d.add = add i64 %d.load, 12931 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("device") monotonic, align 82932 2933 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("device") monotonic, align 42934 %e.add = fadd float %e.load, 1.2935 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("device") monotonic, align 42936 2937 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("device") monotonic, align 82938 %f.add = fadd double %f.load, 1.2939 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("device") monotonic, align 82940 2941 ret void2942}2943 2944; CHECK-LABEL: shared_monotonic_cta2945define void @shared_monotonic_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {2946; CHECK-LABEL: shared_monotonic_cta(2947; CHECK: {2948; CHECK-NEXT: .reg .b16 %rs<5>;2949; CHECK-NEXT: .reg .b32 %r<5>;2950; CHECK-NEXT: .reg .b64 %rd<10>;2951; CHECK-EMPTY:2952; CHECK-NEXT: // %bb.0:2953; CHECK-NEXT: ld.param.b64 %rd1, [shared_monotonic_cta_param_0];2954; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];2955; CHECK-NEXT: ld.param.b64 %rd2, [shared_monotonic_cta_param_1];2956; CHECK-NEXT: add.s16 %rs2, %rs1, 1;2957; CHECK-NEXT: ld.param.b64 %rd3, [shared_monotonic_cta_param_2];2958; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs2;2959; CHECK-NEXT: ld.param.b64 %rd4, [shared_monotonic_cta_param_3];2960; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs3, [%rd2];2961; CHECK-NEXT: ld.param.b64 %rd5, [shared_monotonic_cta_param_4];2962; CHECK-NEXT: add.s16 %rs4, %rs3, 1;2963; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd2], %rs4;2964; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd3];2965; CHECK-NEXT: add.s32 %r2, %r1, 1;2966; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd3], %r2;2967; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd6, [%rd4];2968; CHECK-NEXT: add.s64 %rd7, %rd6, 1;2969; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd4], %rd7;2970; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r3, [%rd5];2971; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;2972; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd5], %r4;2973; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd8, [%rd5];2974; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;2975; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd5], %rd9;2976; CHECK-NEXT: ret;2977 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("block") monotonic, align 12978 %a.add = add i8 %a.load, 12979 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("block") monotonic, align 12980 2981 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("block") monotonic, align 22982 %b.add = add i16 %b.load, 12983 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("block") monotonic, align 22984 2985 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("block") monotonic, align 42986 %c.add = add i32 %c.load, 12987 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("block") monotonic, align 42988 2989 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("block") monotonic, align 82990 %d.add = add i64 %d.load, 12991 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("block") monotonic, align 82992 2993 %e.load = load atomic float, ptr addrspace(3) %e syncscope("block") monotonic, align 42994 %e.add = fadd float %e.load, 1.2995 store atomic float %e.add, ptr addrspace(3) %e syncscope("block") monotonic, align 42996 2997 %f.load = load atomic double, ptr addrspace(3) %e syncscope("block") monotonic, align 82998 %f.add = fadd double %f.load, 1.2999 store atomic double %f.add, ptr addrspace(3) %e syncscope("block") monotonic, align 83000 3001 ret void3002}3003 3004; CHECK-LABEL: shared_monotonic_volatile_cta3005define void @shared_monotonic_volatile_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3006; CHECK-LABEL: shared_monotonic_volatile_cta(3007; CHECK: {3008; CHECK-NEXT: .reg .b16 %rs<5>;3009; CHECK-NEXT: .reg .b32 %r<5>;3010; CHECK-NEXT: .reg .b64 %rd<10>;3011; CHECK-EMPTY:3012; CHECK-NEXT: // %bb.0:3013; CHECK-NEXT: ld.param.b64 %rd1, [shared_monotonic_volatile_cta_param_0];3014; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];3015; CHECK-NEXT: ld.param.b64 %rd2, [shared_monotonic_volatile_cta_param_1];3016; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3017; CHECK-NEXT: ld.param.b64 %rd3, [shared_monotonic_volatile_cta_param_2];3018; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;3019; CHECK-NEXT: ld.param.b64 %rd4, [shared_monotonic_volatile_cta_param_3];3020; CHECK-NEXT: ld.volatile.shared.b16 %rs3, [%rd2];3021; CHECK-NEXT: ld.param.b64 %rd5, [shared_monotonic_volatile_cta_param_4];3022; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3023; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;3024; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd3];3025; CHECK-NEXT: add.s32 %r2, %r1, 1;3026; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;3027; CHECK-NEXT: ld.volatile.shared.b64 %rd6, [%rd4];3028; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3029; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;3030; CHECK-NEXT: ld.volatile.shared.b32 %r3, [%rd5];3031; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3032; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;3033; CHECK-NEXT: ld.volatile.shared.b64 %rd8, [%rd5];3034; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3035; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;3036; CHECK-NEXT: ret;3037 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("block") monotonic, align 13038 %a.add = add i8 %a.load, 13039 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("block") monotonic, align 13040 3041 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("block") monotonic, align 23042 %b.add = add i16 %b.load, 13043 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("block") monotonic, align 23044 3045 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("block") monotonic, align 43046 %c.add = add i32 %c.load, 13047 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("block") monotonic, align 43048 3049 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("block") monotonic, align 83050 %d.add = add i64 %d.load, 13051 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("block") monotonic, align 83052 3053 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("block") monotonic, align 43054 %e.add = fadd float %e.load, 1.3055 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("block") monotonic, align 43056 3057 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("block") monotonic, align 83058 %f.add = fadd double %f.load, 1.3059 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("block") monotonic, align 83060 3061 ret void3062}3063 3064; CHECK-LABEL: shared_acq_rel_sys3065define void @shared_acq_rel_sys(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3066; CHECK-LABEL: shared_acq_rel_sys(3067; CHECK: {3068; CHECK-NEXT: .reg .b16 %rs<5>;3069; CHECK-NEXT: .reg .b32 %r<5>;3070; CHECK-NEXT: .reg .b64 %rd<10>;3071; CHECK-EMPTY:3072; CHECK-NEXT: // %bb.0:3073; CHECK-NEXT: ld.param.b64 %rd1, [shared_acq_rel_sys_param_0];3074; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3075; CHECK-NEXT: ld.param.b64 %rd2, [shared_acq_rel_sys_param_1];3076; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3077; CHECK-NEXT: ld.param.b64 %rd3, [shared_acq_rel_sys_param_2];3078; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3079; CHECK-NEXT: ld.param.b64 %rd4, [shared_acq_rel_sys_param_3];3080; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3081; CHECK-NEXT: ld.param.b64 %rd5, [shared_acq_rel_sys_param_4];3082; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3083; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3084; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3085; CHECK-NEXT: add.s32 %r2, %r1, 1;3086; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3087; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3088; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3089; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3090; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3091; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3092; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3093; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3094; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3095; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3096; CHECK-NEXT: ret;3097 %a.load = load atomic i8, ptr addrspace(3) %a acquire, align 13098 %a.add = add i8 %a.load, 13099 store atomic i8 %a.add, ptr addrspace(3) %a release, align 13100 3101 %b.load = load atomic i16, ptr addrspace(3) %b acquire, align 23102 %b.add = add i16 %b.load, 13103 store atomic i16 %b.add, ptr addrspace(3) %b release, align 23104 3105 %c.load = load atomic i32, ptr addrspace(3) %c acquire, align 43106 %c.add = add i32 %c.load, 13107 store atomic i32 %c.add, ptr addrspace(3) %c release, align 43108 3109 %d.load = load atomic i64, ptr addrspace(3) %d acquire, align 83110 %d.add = add i64 %d.load, 13111 store atomic i64 %d.add, ptr addrspace(3) %d release, align 83112 3113 %e.load = load atomic float, ptr addrspace(3) %e acquire, align 43114 %e.add = fadd float %e.load, 1.3115 store atomic float %e.add, ptr addrspace(3) %e release, align 43116 3117 %f.load = load atomic double, ptr addrspace(3) %e acquire, align 83118 %f.add = fadd double %f.load, 1.3119 store atomic double %f.add, ptr addrspace(3) %e release, align 83120 3121 ret void3122}3123 3124; CHECK-LABEL: shared_acq_rel_volatile_sys3125define void @shared_acq_rel_volatile_sys(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3126; CHECK-LABEL: shared_acq_rel_volatile_sys(3127; CHECK: {3128; CHECK-NEXT: .reg .b16 %rs<5>;3129; CHECK-NEXT: .reg .b32 %r<5>;3130; CHECK-NEXT: .reg .b64 %rd<10>;3131; CHECK-EMPTY:3132; CHECK-NEXT: // %bb.0:3133; CHECK-NEXT: ld.param.b64 %rd1, [shared_acq_rel_volatile_sys_param_0];3134; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3135; CHECK-NEXT: ld.param.b64 %rd2, [shared_acq_rel_volatile_sys_param_1];3136; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3137; CHECK-NEXT: ld.param.b64 %rd3, [shared_acq_rel_volatile_sys_param_2];3138; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3139; CHECK-NEXT: ld.param.b64 %rd4, [shared_acq_rel_volatile_sys_param_3];3140; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3141; CHECK-NEXT: ld.param.b64 %rd5, [shared_acq_rel_volatile_sys_param_4];3142; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3143; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3144; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3145; CHECK-NEXT: add.s32 %r2, %r1, 1;3146; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3147; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3148; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3149; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3150; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3151; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3152; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3153; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3154; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3155; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3156; CHECK-NEXT: ret;3157 %a.load = load atomic volatile i8, ptr addrspace(3) %a acquire, align 13158 %a.add = add i8 %a.load, 13159 store atomic volatile i8 %a.add, ptr addrspace(3) %a release, align 13160 3161 %b.load = load atomic volatile i16, ptr addrspace(3) %b acquire, align 23162 %b.add = add i16 %b.load, 13163 store atomic volatile i16 %b.add, ptr addrspace(3) %b release, align 23164 3165 %c.load = load atomic volatile i32, ptr addrspace(3) %c acquire, align 43166 %c.add = add i32 %c.load, 13167 store atomic volatile i32 %c.add, ptr addrspace(3) %c release, align 43168 3169 %d.load = load atomic volatile i64, ptr addrspace(3) %d acquire, align 83170 %d.add = add i64 %d.load, 13171 store atomic volatile i64 %d.add, ptr addrspace(3) %d release, align 83172 3173 %e.load = load atomic volatile float, ptr addrspace(3) %e acquire, align 43174 %e.add = fadd float %e.load, 1.3175 store atomic volatile float %e.add, ptr addrspace(3) %e release, align 43176 3177 %f.load = load atomic volatile double, ptr addrspace(3) %e acquire, align 83178 %f.add = fadd double %f.load, 1.3179 store atomic volatile double %f.add, ptr addrspace(3) %e release, align 83180 3181 ret void3182}3183 3184; CHECK-LABEL: shared_acq_rel_gpu3185define void @shared_acq_rel_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3186; CHECK-LABEL: shared_acq_rel_gpu(3187; CHECK: {3188; CHECK-NEXT: .reg .b16 %rs<5>;3189; CHECK-NEXT: .reg .b32 %r<5>;3190; CHECK-NEXT: .reg .b64 %rd<10>;3191; CHECK-EMPTY:3192; CHECK-NEXT: // %bb.0:3193; CHECK-NEXT: ld.param.b64 %rd1, [shared_acq_rel_gpu_param_0];3194; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];3195; CHECK-NEXT: ld.param.b64 %rd2, [shared_acq_rel_gpu_param_1];3196; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3197; CHECK-NEXT: ld.param.b64 %rd3, [shared_acq_rel_gpu_param_2];3198; CHECK-NEXT: st.release.gpu.shared.b8 [%rd1], %rs2;3199; CHECK-NEXT: ld.param.b64 %rd4, [shared_acq_rel_gpu_param_3];3200; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs3, [%rd2];3201; CHECK-NEXT: ld.param.b64 %rd5, [shared_acq_rel_gpu_param_4];3202; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3203; CHECK-NEXT: st.release.gpu.shared.b16 [%rd2], %rs4;3204; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd3];3205; CHECK-NEXT: add.s32 %r2, %r1, 1;3206; CHECK-NEXT: st.release.gpu.shared.b32 [%rd3], %r2;3207; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd6, [%rd4];3208; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3209; CHECK-NEXT: st.release.gpu.shared.b64 [%rd4], %rd7;3210; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r3, [%rd5];3211; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3212; CHECK-NEXT: st.release.gpu.shared.b32 [%rd5], %r4;3213; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd8, [%rd5];3214; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3215; CHECK-NEXT: st.release.gpu.shared.b64 [%rd5], %rd9;3216; CHECK-NEXT: ret;3217 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("device") acquire, align 13218 %a.add = add i8 %a.load, 13219 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("device") release, align 13220 3221 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("device") acquire, align 23222 %b.add = add i16 %b.load, 13223 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("device") release, align 23224 3225 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("device") acquire, align 43226 %c.add = add i32 %c.load, 13227 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("device") release, align 43228 3229 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("device") acquire, align 83230 %d.add = add i64 %d.load, 13231 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("device") release, align 83232 3233 %e.load = load atomic float, ptr addrspace(3) %e syncscope("device") acquire, align 43234 %e.add = fadd float %e.load, 1.3235 store atomic float %e.add, ptr addrspace(3) %e syncscope("device") release, align 43236 3237 %f.load = load atomic double, ptr addrspace(3) %e syncscope("device") acquire, align 83238 %f.add = fadd double %f.load, 1.3239 store atomic double %f.add, ptr addrspace(3) %e syncscope("device") release, align 83240 3241 ret void3242}3243 3244; CHECK-LABEL: shared_acq_rel_volatile_gpu3245define void @shared_acq_rel_volatile_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3246; CHECK-LABEL: shared_acq_rel_volatile_gpu(3247; CHECK: {3248; CHECK-NEXT: .reg .b16 %rs<5>;3249; CHECK-NEXT: .reg .b32 %r<5>;3250; CHECK-NEXT: .reg .b64 %rd<10>;3251; CHECK-EMPTY:3252; CHECK-NEXT: // %bb.0:3253; CHECK-NEXT: ld.param.b64 %rd1, [shared_acq_rel_volatile_gpu_param_0];3254; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3255; CHECK-NEXT: ld.param.b64 %rd2, [shared_acq_rel_volatile_gpu_param_1];3256; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3257; CHECK-NEXT: ld.param.b64 %rd3, [shared_acq_rel_volatile_gpu_param_2];3258; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3259; CHECK-NEXT: ld.param.b64 %rd4, [shared_acq_rel_volatile_gpu_param_3];3260; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3261; CHECK-NEXT: ld.param.b64 %rd5, [shared_acq_rel_volatile_gpu_param_4];3262; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3263; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3264; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3265; CHECK-NEXT: add.s32 %r2, %r1, 1;3266; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3267; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3268; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3269; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3270; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3271; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3272; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3273; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3274; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3275; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3276; CHECK-NEXT: ret;3277 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("device") acquire, align 13278 %a.add = add i8 %a.load, 13279 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("device") release, align 13280 3281 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("device") acquire, align 23282 %b.add = add i16 %b.load, 13283 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("device") release, align 23284 3285 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("device") acquire, align 43286 %c.add = add i32 %c.load, 13287 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("device") release, align 43288 3289 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("device") acquire, align 83290 %d.add = add i64 %d.load, 13291 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("device") release, align 83292 3293 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("device") acquire, align 43294 %e.add = fadd float %e.load, 1.3295 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("device") release, align 43296 3297 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("device") acquire, align 83298 %f.add = fadd double %f.load, 1.3299 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("device") release, align 83300 3301 ret void3302}3303 3304; CHECK-LABEL: shared_acq_rel_cta3305define void @shared_acq_rel_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3306; CHECK-LABEL: shared_acq_rel_cta(3307; CHECK: {3308; CHECK-NEXT: .reg .b16 %rs<5>;3309; CHECK-NEXT: .reg .b32 %r<5>;3310; CHECK-NEXT: .reg .b64 %rd<10>;3311; CHECK-EMPTY:3312; CHECK-NEXT: // %bb.0:3313; CHECK-NEXT: ld.param.b64 %rd1, [shared_acq_rel_cta_param_0];3314; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];3315; CHECK-NEXT: ld.param.b64 %rd2, [shared_acq_rel_cta_param_1];3316; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3317; CHECK-NEXT: ld.param.b64 %rd3, [shared_acq_rel_cta_param_2];3318; CHECK-NEXT: st.release.cta.shared.b8 [%rd1], %rs2;3319; CHECK-NEXT: ld.param.b64 %rd4, [shared_acq_rel_cta_param_3];3320; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs3, [%rd2];3321; CHECK-NEXT: ld.param.b64 %rd5, [shared_acq_rel_cta_param_4];3322; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3323; CHECK-NEXT: st.release.cta.shared.b16 [%rd2], %rs4;3324; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd3];3325; CHECK-NEXT: add.s32 %r2, %r1, 1;3326; CHECK-NEXT: st.release.cta.shared.b32 [%rd3], %r2;3327; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd6, [%rd4];3328; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3329; CHECK-NEXT: st.release.cta.shared.b64 [%rd4], %rd7;3330; CHECK-NEXT: ld.acquire.cta.shared.b32 %r3, [%rd5];3331; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3332; CHECK-NEXT: st.release.cta.shared.b32 [%rd5], %r4;3333; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd8, [%rd5];3334; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3335; CHECK-NEXT: st.release.cta.shared.b64 [%rd5], %rd9;3336; CHECK-NEXT: ret;3337 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("block") acquire, align 13338 %a.add = add i8 %a.load, 13339 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("block") release, align 13340 3341 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("block") acquire, align 23342 %b.add = add i16 %b.load, 13343 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("block") release, align 23344 3345 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("block") acquire, align 43346 %c.add = add i32 %c.load, 13347 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("block") release, align 43348 3349 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("block") acquire, align 83350 %d.add = add i64 %d.load, 13351 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("block") release, align 83352 3353 %e.load = load atomic float, ptr addrspace(3) %e syncscope("block") acquire, align 43354 %e.add = fadd float %e.load, 1.3355 store atomic float %e.add, ptr addrspace(3) %e syncscope("block") release, align 43356 3357 %f.load = load atomic double, ptr addrspace(3) %e syncscope("block") acquire, align 83358 %f.add = fadd double %f.load, 1.3359 store atomic double %f.add, ptr addrspace(3) %e syncscope("block") release, align 83360 3361 ret void3362}3363 3364; CHECK-LABEL: shared_acq_rel_volatile_cta3365define void @shared_acq_rel_volatile_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3366; CHECK-LABEL: shared_acq_rel_volatile_cta(3367; CHECK: {3368; CHECK-NEXT: .reg .b16 %rs<5>;3369; CHECK-NEXT: .reg .b32 %r<5>;3370; CHECK-NEXT: .reg .b64 %rd<10>;3371; CHECK-EMPTY:3372; CHECK-NEXT: // %bb.0:3373; CHECK-NEXT: ld.param.b64 %rd1, [shared_acq_rel_volatile_cta_param_0];3374; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3375; CHECK-NEXT: ld.param.b64 %rd2, [shared_acq_rel_volatile_cta_param_1];3376; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3377; CHECK-NEXT: ld.param.b64 %rd3, [shared_acq_rel_volatile_cta_param_2];3378; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3379; CHECK-NEXT: ld.param.b64 %rd4, [shared_acq_rel_volatile_cta_param_3];3380; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3381; CHECK-NEXT: ld.param.b64 %rd5, [shared_acq_rel_volatile_cta_param_4];3382; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3383; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3384; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3385; CHECK-NEXT: add.s32 %r2, %r1, 1;3386; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3387; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3388; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3389; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3390; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3391; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3392; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3393; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3394; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3395; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3396; CHECK-NEXT: ret;3397 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("block") acquire, align 13398 %a.add = add i8 %a.load, 13399 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("block") release, align 13400 3401 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("block") acquire, align 23402 %b.add = add i16 %b.load, 13403 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("block") release, align 23404 3405 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("block") acquire, align 43406 %c.add = add i32 %c.load, 13407 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("block") release, align 43408 3409 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("block") acquire, align 83410 %d.add = add i64 %d.load, 13411 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("block") release, align 83412 3413 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("block") acquire, align 43414 %e.add = fadd float %e.load, 1.3415 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("block") release, align 43416 3417 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("block") acquire, align 83418 %f.add = fadd double %f.load, 1.3419 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("block") release, align 83420 3421 ret void3422}3423 3424; CHECK-LABEL: shared_seq_cst_sys3425define void @shared_seq_cst_sys(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3426; CHECK-LABEL: shared_seq_cst_sys(3427; CHECK: {3428; CHECK-NEXT: .reg .b16 %rs<5>;3429; CHECK-NEXT: .reg .b32 %r<5>;3430; CHECK-NEXT: .reg .b64 %rd<10>;3431; CHECK-EMPTY:3432; CHECK-NEXT: // %bb.0:3433; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_sys_param_0];3434; CHECK-NEXT: fence.sc.sys;3435; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3436; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_sys_param_1];3437; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3438; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_sys_param_2];3439; CHECK-NEXT: fence.sc.sys;3440; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3441; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_sys_param_3];3442; CHECK-NEXT: fence.sc.sys;3443; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3444; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_sys_param_4];3445; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3446; CHECK-NEXT: fence.sc.sys;3447; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3448; CHECK-NEXT: fence.sc.sys;3449; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3450; CHECK-NEXT: add.s32 %r2, %r1, 1;3451; CHECK-NEXT: fence.sc.sys;3452; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3453; CHECK-NEXT: fence.sc.sys;3454; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3455; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3456; CHECK-NEXT: fence.sc.sys;3457; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3458; CHECK-NEXT: fence.sc.sys;3459; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3460; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3461; CHECK-NEXT: fence.sc.sys;3462; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3463; CHECK-NEXT: fence.sc.sys;3464; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3465; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3466; CHECK-NEXT: fence.sc.sys;3467; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3468; CHECK-NEXT: ret;3469 %a.load = load atomic i8, ptr addrspace(3) %a seq_cst, align 13470 %a.add = add i8 %a.load, 13471 store atomic i8 %a.add, ptr addrspace(3) %a seq_cst, align 13472 3473 %b.load = load atomic i16, ptr addrspace(3) %b seq_cst, align 23474 %b.add = add i16 %b.load, 13475 store atomic i16 %b.add, ptr addrspace(3) %b seq_cst, align 23476 3477 %c.load = load atomic i32, ptr addrspace(3) %c seq_cst, align 43478 %c.add = add i32 %c.load, 13479 store atomic i32 %c.add, ptr addrspace(3) %c seq_cst, align 43480 3481 %d.load = load atomic i64, ptr addrspace(3) %d seq_cst, align 83482 %d.add = add i64 %d.load, 13483 store atomic i64 %d.add, ptr addrspace(3) %d seq_cst, align 83484 3485 %e.load = load atomic float, ptr addrspace(3) %e seq_cst, align 43486 %e.add = fadd float %e.load, 1.3487 store atomic float %e.add, ptr addrspace(3) %e seq_cst, align 43488 3489 %f.load = load atomic double, ptr addrspace(3) %e seq_cst, align 83490 %f.add = fadd double %f.load, 1.3491 store atomic double %f.add, ptr addrspace(3) %e seq_cst, align 83492 3493 ret void3494}3495 3496; CHECK-LABEL: shared_seq_cst_volatile_sys3497define void @shared_seq_cst_volatile_sys(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3498; CHECK-LABEL: shared_seq_cst_volatile_sys(3499; CHECK: {3500; CHECK-NEXT: .reg .b16 %rs<5>;3501; CHECK-NEXT: .reg .b32 %r<5>;3502; CHECK-NEXT: .reg .b64 %rd<10>;3503; CHECK-EMPTY:3504; CHECK-NEXT: // %bb.0:3505; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_volatile_sys_param_0];3506; CHECK-NEXT: fence.sc.sys;3507; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3508; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_volatile_sys_param_1];3509; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3510; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_volatile_sys_param_2];3511; CHECK-NEXT: fence.sc.sys;3512; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3513; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_volatile_sys_param_3];3514; CHECK-NEXT: fence.sc.sys;3515; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3516; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_volatile_sys_param_4];3517; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3518; CHECK-NEXT: fence.sc.sys;3519; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3520; CHECK-NEXT: fence.sc.sys;3521; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3522; CHECK-NEXT: add.s32 %r2, %r1, 1;3523; CHECK-NEXT: fence.sc.sys;3524; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3525; CHECK-NEXT: fence.sc.sys;3526; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3527; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3528; CHECK-NEXT: fence.sc.sys;3529; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3530; CHECK-NEXT: fence.sc.sys;3531; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3532; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3533; CHECK-NEXT: fence.sc.sys;3534; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3535; CHECK-NEXT: fence.sc.sys;3536; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3537; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3538; CHECK-NEXT: fence.sc.sys;3539; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3540; CHECK-NEXT: ret;3541 %a.load = load atomic volatile i8, ptr addrspace(3) %a seq_cst, align 13542 %a.add = add i8 %a.load, 13543 store atomic volatile i8 %a.add, ptr addrspace(3) %a seq_cst, align 13544 3545 %b.load = load atomic volatile i16, ptr addrspace(3) %b seq_cst, align 23546 %b.add = add i16 %b.load, 13547 store atomic volatile i16 %b.add, ptr addrspace(3) %b seq_cst, align 23548 3549 %c.load = load atomic volatile i32, ptr addrspace(3) %c seq_cst, align 43550 %c.add = add i32 %c.load, 13551 store atomic volatile i32 %c.add, ptr addrspace(3) %c seq_cst, align 43552 3553 %d.load = load atomic volatile i64, ptr addrspace(3) %d seq_cst, align 83554 %d.add = add i64 %d.load, 13555 store atomic volatile i64 %d.add, ptr addrspace(3) %d seq_cst, align 83556 3557 %e.load = load atomic volatile float, ptr addrspace(3) %e seq_cst, align 43558 %e.add = fadd float %e.load, 1.3559 store atomic volatile float %e.add, ptr addrspace(3) %e seq_cst, align 43560 3561 %f.load = load atomic volatile double, ptr addrspace(3) %e seq_cst, align 83562 %f.add = fadd double %f.load, 1.3563 store atomic volatile double %f.add, ptr addrspace(3) %e seq_cst, align 83564 3565 ret void3566}3567 3568; CHECK-LABEL: shared_seq_cst_gpu3569define void @shared_seq_cst_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3570; CHECK-LABEL: shared_seq_cst_gpu(3571; CHECK: {3572; CHECK-NEXT: .reg .b16 %rs<5>;3573; CHECK-NEXT: .reg .b32 %r<5>;3574; CHECK-NEXT: .reg .b64 %rd<10>;3575; CHECK-EMPTY:3576; CHECK-NEXT: // %bb.0:3577; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_gpu_param_0];3578; CHECK-NEXT: fence.sc.gpu;3579; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];3580; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_gpu_param_1];3581; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3582; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_gpu_param_2];3583; CHECK-NEXT: fence.sc.gpu;3584; CHECK-NEXT: st.release.gpu.shared.b8 [%rd1], %rs2;3585; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_gpu_param_3];3586; CHECK-NEXT: fence.sc.gpu;3587; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs3, [%rd2];3588; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_gpu_param_4];3589; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3590; CHECK-NEXT: fence.sc.gpu;3591; CHECK-NEXT: st.release.gpu.shared.b16 [%rd2], %rs4;3592; CHECK-NEXT: fence.sc.gpu;3593; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd3];3594; CHECK-NEXT: add.s32 %r2, %r1, 1;3595; CHECK-NEXT: fence.sc.gpu;3596; CHECK-NEXT: st.release.gpu.shared.b32 [%rd3], %r2;3597; CHECK-NEXT: fence.sc.gpu;3598; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd6, [%rd4];3599; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3600; CHECK-NEXT: fence.sc.gpu;3601; CHECK-NEXT: st.release.gpu.shared.b64 [%rd4], %rd7;3602; CHECK-NEXT: fence.sc.gpu;3603; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r3, [%rd5];3604; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3605; CHECK-NEXT: fence.sc.gpu;3606; CHECK-NEXT: st.release.gpu.shared.b32 [%rd5], %r4;3607; CHECK-NEXT: fence.sc.gpu;3608; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd8, [%rd5];3609; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3610; CHECK-NEXT: fence.sc.gpu;3611; CHECK-NEXT: st.release.gpu.shared.b64 [%rd5], %rd9;3612; CHECK-NEXT: ret;3613 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("device") seq_cst, align 13614 %a.add = add i8 %a.load, 13615 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("device") seq_cst, align 13616 3617 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("device") seq_cst, align 23618 %b.add = add i16 %b.load, 13619 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("device") seq_cst, align 23620 3621 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("device") seq_cst, align 43622 %c.add = add i32 %c.load, 13623 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("device") seq_cst, align 43624 3625 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("device") seq_cst, align 83626 %d.add = add i64 %d.load, 13627 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("device") seq_cst, align 83628 3629 %e.load = load atomic float, ptr addrspace(3) %e syncscope("device") seq_cst, align 43630 %e.add = fadd float %e.load, 1.3631 store atomic float %e.add, ptr addrspace(3) %e syncscope("device") seq_cst, align 43632 3633 %f.load = load atomic double, ptr addrspace(3) %e syncscope("device") seq_cst, align 83634 %f.add = fadd double %f.load, 1.3635 store atomic double %f.add, ptr addrspace(3) %e syncscope("device") seq_cst, align 83636 3637 ret void3638}3639 3640; CHECK-LABEL: shared_seq_cst_volatile_gpu3641define void @shared_seq_cst_volatile_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3642; CHECK-LABEL: shared_seq_cst_volatile_gpu(3643; CHECK: {3644; CHECK-NEXT: .reg .b16 %rs<5>;3645; CHECK-NEXT: .reg .b32 %r<5>;3646; CHECK-NEXT: .reg .b64 %rd<10>;3647; CHECK-EMPTY:3648; CHECK-NEXT: // %bb.0:3649; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_volatile_gpu_param_0];3650; CHECK-NEXT: fence.sc.sys;3651; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3652; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_volatile_gpu_param_1];3653; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3654; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_volatile_gpu_param_2];3655; CHECK-NEXT: fence.sc.sys;3656; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3657; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_volatile_gpu_param_3];3658; CHECK-NEXT: fence.sc.sys;3659; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3660; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_volatile_gpu_param_4];3661; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3662; CHECK-NEXT: fence.sc.sys;3663; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3664; CHECK-NEXT: fence.sc.sys;3665; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3666; CHECK-NEXT: add.s32 %r2, %r1, 1;3667; CHECK-NEXT: fence.sc.sys;3668; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3669; CHECK-NEXT: fence.sc.sys;3670; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3671; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3672; CHECK-NEXT: fence.sc.sys;3673; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3674; CHECK-NEXT: fence.sc.sys;3675; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3676; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3677; CHECK-NEXT: fence.sc.sys;3678; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3679; CHECK-NEXT: fence.sc.sys;3680; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3681; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3682; CHECK-NEXT: fence.sc.sys;3683; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3684; CHECK-NEXT: ret;3685 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("device") seq_cst, align 13686 %a.add = add i8 %a.load, 13687 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("device") seq_cst, align 13688 3689 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("device") seq_cst, align 23690 %b.add = add i16 %b.load, 13691 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("device") seq_cst, align 23692 3693 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("device") seq_cst, align 43694 %c.add = add i32 %c.load, 13695 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("device") seq_cst, align 43696 3697 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("device") seq_cst, align 83698 %d.add = add i64 %d.load, 13699 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("device") seq_cst, align 83700 3701 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("device") seq_cst, align 43702 %e.add = fadd float %e.load, 1.3703 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("device") seq_cst, align 43704 3705 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("device") seq_cst, align 83706 %f.add = fadd double %f.load, 1.3707 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("device") seq_cst, align 83708 3709 ret void3710}3711 3712; CHECK-LABEL: shared_seq_cst_cta3713define void @shared_seq_cst_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3714; CHECK-LABEL: shared_seq_cst_cta(3715; CHECK: {3716; CHECK-NEXT: .reg .b16 %rs<5>;3717; CHECK-NEXT: .reg .b32 %r<5>;3718; CHECK-NEXT: .reg .b64 %rd<10>;3719; CHECK-EMPTY:3720; CHECK-NEXT: // %bb.0:3721; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_cta_param_0];3722; CHECK-NEXT: fence.sc.cta;3723; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];3724; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_cta_param_1];3725; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3726; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_cta_param_2];3727; CHECK-NEXT: fence.sc.cta;3728; CHECK-NEXT: st.release.cta.shared.b8 [%rd1], %rs2;3729; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_cta_param_3];3730; CHECK-NEXT: fence.sc.cta;3731; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs3, [%rd2];3732; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_cta_param_4];3733; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3734; CHECK-NEXT: fence.sc.cta;3735; CHECK-NEXT: st.release.cta.shared.b16 [%rd2], %rs4;3736; CHECK-NEXT: fence.sc.cta;3737; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd3];3738; CHECK-NEXT: add.s32 %r2, %r1, 1;3739; CHECK-NEXT: fence.sc.cta;3740; CHECK-NEXT: st.release.cta.shared.b32 [%rd3], %r2;3741; CHECK-NEXT: fence.sc.cta;3742; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd6, [%rd4];3743; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3744; CHECK-NEXT: fence.sc.cta;3745; CHECK-NEXT: st.release.cta.shared.b64 [%rd4], %rd7;3746; CHECK-NEXT: fence.sc.cta;3747; CHECK-NEXT: ld.acquire.cta.shared.b32 %r3, [%rd5];3748; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3749; CHECK-NEXT: fence.sc.cta;3750; CHECK-NEXT: st.release.cta.shared.b32 [%rd5], %r4;3751; CHECK-NEXT: fence.sc.cta;3752; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd8, [%rd5];3753; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3754; CHECK-NEXT: fence.sc.cta;3755; CHECK-NEXT: st.release.cta.shared.b64 [%rd5], %rd9;3756; CHECK-NEXT: ret;3757 %a.load = load atomic i8, ptr addrspace(3) %a syncscope("block") seq_cst, align 13758 %a.add = add i8 %a.load, 13759 store atomic i8 %a.add, ptr addrspace(3) %a syncscope("block") seq_cst, align 13760 3761 %b.load = load atomic i16, ptr addrspace(3) %b syncscope("block") seq_cst, align 23762 %b.add = add i16 %b.load, 13763 store atomic i16 %b.add, ptr addrspace(3) %b syncscope("block") seq_cst, align 23764 3765 %c.load = load atomic i32, ptr addrspace(3) %c syncscope("block") seq_cst, align 43766 %c.add = add i32 %c.load, 13767 store atomic i32 %c.add, ptr addrspace(3) %c syncscope("block") seq_cst, align 43768 3769 %d.load = load atomic i64, ptr addrspace(3) %d syncscope("block") seq_cst, align 83770 %d.add = add i64 %d.load, 13771 store atomic i64 %d.add, ptr addrspace(3) %d syncscope("block") seq_cst, align 83772 3773 %e.load = load atomic float, ptr addrspace(3) %e syncscope("block") seq_cst, align 43774 %e.add = fadd float %e.load, 1.3775 store atomic float %e.add, ptr addrspace(3) %e syncscope("block") seq_cst, align 43776 3777 %f.load = load atomic double, ptr addrspace(3) %e syncscope("block") seq_cst, align 83778 %f.add = fadd double %f.load, 1.3779 store atomic double %f.add, ptr addrspace(3) %e syncscope("block") seq_cst, align 83780 3781 ret void3782}3783 3784; CHECK-LABEL: shared_seq_cst_volatile_cta3785define void @shared_seq_cst_volatile_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d, ptr addrspace(3) %e) local_unnamed_addr {3786; CHECK-LABEL: shared_seq_cst_volatile_cta(3787; CHECK: {3788; CHECK-NEXT: .reg .b16 %rs<5>;3789; CHECK-NEXT: .reg .b32 %r<5>;3790; CHECK-NEXT: .reg .b64 %rd<10>;3791; CHECK-EMPTY:3792; CHECK-NEXT: // %bb.0:3793; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_volatile_cta_param_0];3794; CHECK-NEXT: fence.sc.sys;3795; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];3796; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_volatile_cta_param_1];3797; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3798; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_volatile_cta_param_2];3799; CHECK-NEXT: fence.sc.sys;3800; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;3801; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_volatile_cta_param_3];3802; CHECK-NEXT: fence.sc.sys;3803; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];3804; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_volatile_cta_param_4];3805; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3806; CHECK-NEXT: fence.sc.sys;3807; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;3808; CHECK-NEXT: fence.sc.sys;3809; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];3810; CHECK-NEXT: add.s32 %r2, %r1, 1;3811; CHECK-NEXT: fence.sc.sys;3812; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;3813; CHECK-NEXT: fence.sc.sys;3814; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];3815; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3816; CHECK-NEXT: fence.sc.sys;3817; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;3818; CHECK-NEXT: fence.sc.sys;3819; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];3820; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3821; CHECK-NEXT: fence.sc.sys;3822; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;3823; CHECK-NEXT: fence.sc.sys;3824; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];3825; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3826; CHECK-NEXT: fence.sc.sys;3827; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;3828; CHECK-NEXT: ret;3829 %a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("block") seq_cst, align 13830 %a.add = add i8 %a.load, 13831 store atomic volatile i8 %a.add, ptr addrspace(3) %a syncscope("block") seq_cst, align 13832 3833 %b.load = load atomic volatile i16, ptr addrspace(3) %b syncscope("block") seq_cst, align 23834 %b.add = add i16 %b.load, 13835 store atomic volatile i16 %b.add, ptr addrspace(3) %b syncscope("block") seq_cst, align 23836 3837 %c.load = load atomic volatile i32, ptr addrspace(3) %c syncscope("block") seq_cst, align 43838 %c.add = add i32 %c.load, 13839 store atomic volatile i32 %c.add, ptr addrspace(3) %c syncscope("block") seq_cst, align 43840 3841 %d.load = load atomic volatile i64, ptr addrspace(3) %d syncscope("block") seq_cst, align 83842 %d.add = add i64 %d.load, 13843 store atomic volatile i64 %d.add, ptr addrspace(3) %d syncscope("block") seq_cst, align 83844 3845 %e.load = load atomic volatile float, ptr addrspace(3) %e syncscope("block") seq_cst, align 43846 %e.add = fadd float %e.load, 1.3847 store atomic volatile float %e.add, ptr addrspace(3) %e syncscope("block") seq_cst, align 43848 3849 %f.load = load atomic volatile double, ptr addrspace(3) %e syncscope("block") seq_cst, align 83850 %f.add = fadd double %f.load, 1.3851 store atomic volatile double %f.add, ptr addrspace(3) %e syncscope("block") seq_cst, align 83852 3853 ret void3854}3855 3856;; local statespace3857 3858; CHECK-LABEL: local_unordered_gpu3859define void @local_unordered_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {3860; CHECK-LABEL: local_unordered_gpu(3861; CHECK: {3862; CHECK-NEXT: .reg .b16 %rs<5>;3863; CHECK-NEXT: .reg .b32 %r<5>;3864; CHECK-NEXT: .reg .b64 %rd<10>;3865; CHECK-EMPTY:3866; CHECK-NEXT: // %bb.0:3867; CHECK-NEXT: ld.param.b64 %rd1, [local_unordered_gpu_param_0];3868; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];3869; CHECK-NEXT: ld.param.b64 %rd2, [local_unordered_gpu_param_1];3870; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3871; CHECK-NEXT: ld.param.b64 %rd3, [local_unordered_gpu_param_2];3872; CHECK-NEXT: st.local.b8 [%rd1], %rs2;3873; CHECK-NEXT: ld.param.b64 %rd4, [local_unordered_gpu_param_3];3874; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];3875; CHECK-NEXT: ld.param.b64 %rd5, [local_unordered_gpu_param_4];3876; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3877; CHECK-NEXT: st.local.b16 [%rd2], %rs4;3878; CHECK-NEXT: ld.local.b32 %r1, [%rd3];3879; CHECK-NEXT: add.s32 %r2, %r1, 1;3880; CHECK-NEXT: st.local.b32 [%rd3], %r2;3881; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];3882; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3883; CHECK-NEXT: st.local.b64 [%rd4], %rd7;3884; CHECK-NEXT: ld.local.b32 %r3, [%rd5];3885; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3886; CHECK-NEXT: st.local.b32 [%rd5], %r4;3887; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];3888; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3889; CHECK-NEXT: st.local.b64 [%rd5], %rd9;3890; CHECK-NEXT: ret;3891 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("device") unordered, align 13892 %a.add = add i8 %a.load, 13893 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("device") unordered, align 13894 3895 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("device") unordered, align 23896 %b.add = add i16 %b.load, 13897 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("device") unordered, align 23898 3899 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("device") unordered, align 43900 %c.add = add i32 %c.load, 13901 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("device") unordered, align 43902 3903 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("device") unordered, align 83904 %d.add = add i64 %d.load, 13905 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("device") unordered, align 83906 3907 %e.load = load atomic float, ptr addrspace(5) %e syncscope("device") unordered, align 43908 %e.add = fadd float %e.load, 1.3909 store atomic float %e.add, ptr addrspace(5) %e syncscope("device") unordered, align 43910 3911 %f.load = load atomic double, ptr addrspace(5) %e syncscope("device") unordered, align 83912 %f.add = fadd double %f.load, 1.3913 store atomic double %f.add, ptr addrspace(5) %e syncscope("device") unordered, align 83914 3915 ret void3916}3917 3918; CHECK-LABEL: local_unordered_volatile_gpu3919define void @local_unordered_volatile_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {3920; CHECK-LABEL: local_unordered_volatile_gpu(3921; CHECK: {3922; CHECK-NEXT: .reg .b16 %rs<5>;3923; CHECK-NEXT: .reg .b32 %r<5>;3924; CHECK-NEXT: .reg .b64 %rd<10>;3925; CHECK-EMPTY:3926; CHECK-NEXT: // %bb.0:3927; CHECK-NEXT: ld.param.b64 %rd1, [local_unordered_volatile_gpu_param_0];3928; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];3929; CHECK-NEXT: ld.param.b64 %rd2, [local_unordered_volatile_gpu_param_1];3930; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3931; CHECK-NEXT: ld.param.b64 %rd3, [local_unordered_volatile_gpu_param_2];3932; CHECK-NEXT: st.local.b8 [%rd1], %rs2;3933; CHECK-NEXT: ld.param.b64 %rd4, [local_unordered_volatile_gpu_param_3];3934; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];3935; CHECK-NEXT: ld.param.b64 %rd5, [local_unordered_volatile_gpu_param_4];3936; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3937; CHECK-NEXT: st.local.b16 [%rd2], %rs4;3938; CHECK-NEXT: ld.local.b32 %r1, [%rd3];3939; CHECK-NEXT: add.s32 %r2, %r1, 1;3940; CHECK-NEXT: st.local.b32 [%rd3], %r2;3941; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];3942; CHECK-NEXT: add.s64 %rd7, %rd6, 1;3943; CHECK-NEXT: st.local.b64 [%rd4], %rd7;3944; CHECK-NEXT: ld.local.b32 %r3, [%rd5];3945; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;3946; CHECK-NEXT: st.local.b32 [%rd5], %r4;3947; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];3948; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;3949; CHECK-NEXT: st.local.b64 [%rd5], %rd9;3950; CHECK-NEXT: ret;3951 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("device") unordered, align 13952 %a.add = add i8 %a.load, 13953 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("device") unordered, align 13954 3955 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("device") unordered, align 23956 %b.add = add i16 %b.load, 13957 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("device") unordered, align 23958 3959 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("device") unordered, align 43960 %c.add = add i32 %c.load, 13961 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("device") unordered, align 43962 3963 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("device") unordered, align 83964 %d.add = add i64 %d.load, 13965 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("device") unordered, align 83966 3967 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("device") unordered, align 43968 %e.add = fadd float %e.load, 1.3969 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("device") unordered, align 43970 3971 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("device") unordered, align 83972 %f.add = fadd double %f.load, 1.3973 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("device") unordered, align 83974 3975 ret void3976}3977 3978; CHECK-LABEL: local_unordered_cta3979define void @local_unordered_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {3980; CHECK-LABEL: local_unordered_cta(3981; CHECK: {3982; CHECK-NEXT: .reg .b16 %rs<5>;3983; CHECK-NEXT: .reg .b32 %r<5>;3984; CHECK-NEXT: .reg .b64 %rd<10>;3985; CHECK-EMPTY:3986; CHECK-NEXT: // %bb.0:3987; CHECK-NEXT: ld.param.b64 %rd1, [local_unordered_cta_param_0];3988; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];3989; CHECK-NEXT: ld.param.b64 %rd2, [local_unordered_cta_param_1];3990; CHECK-NEXT: add.s16 %rs2, %rs1, 1;3991; CHECK-NEXT: ld.param.b64 %rd3, [local_unordered_cta_param_2];3992; CHECK-NEXT: st.local.b8 [%rd1], %rs2;3993; CHECK-NEXT: ld.param.b64 %rd4, [local_unordered_cta_param_3];3994; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];3995; CHECK-NEXT: ld.param.b64 %rd5, [local_unordered_cta_param_4];3996; CHECK-NEXT: add.s16 %rs4, %rs3, 1;3997; CHECK-NEXT: st.local.b16 [%rd2], %rs4;3998; CHECK-NEXT: ld.local.b32 %r1, [%rd3];3999; CHECK-NEXT: add.s32 %r2, %r1, 1;4000; CHECK-NEXT: st.local.b32 [%rd3], %r2;4001; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4002; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4003; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4004; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4005; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4006; CHECK-NEXT: st.local.b32 [%rd5], %r4;4007; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4008; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4009; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4010; CHECK-NEXT: ret;4011 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("block") unordered, align 14012 %a.add = add i8 %a.load, 14013 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("block") unordered, align 14014 4015 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("block") unordered, align 24016 %b.add = add i16 %b.load, 14017 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("block") unordered, align 24018 4019 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("block") unordered, align 44020 %c.add = add i32 %c.load, 14021 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("block") unordered, align 44022 4023 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("block") unordered, align 84024 %d.add = add i64 %d.load, 14025 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("block") unordered, align 84026 4027 %e.load = load atomic float, ptr addrspace(5) %e syncscope("block") unordered, align 44028 %e.add = fadd float %e.load, 1.4029 store atomic float %e.add, ptr addrspace(5) %e syncscope("block") unordered, align 44030 4031 %f.load = load atomic double, ptr addrspace(5) %e syncscope("block") unordered, align 84032 %f.add = fadd double %f.load, 1.4033 store atomic double %f.add, ptr addrspace(5) %e syncscope("block") unordered, align 84034 4035 ret void4036}4037 4038; CHECK-LABEL: local_unordered_volatile_cta4039define void @local_unordered_volatile_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4040; CHECK-LABEL: local_unordered_volatile_cta(4041; CHECK: {4042; CHECK-NEXT: .reg .b16 %rs<5>;4043; CHECK-NEXT: .reg .b32 %r<5>;4044; CHECK-NEXT: .reg .b64 %rd<10>;4045; CHECK-EMPTY:4046; CHECK-NEXT: // %bb.0:4047; CHECK-NEXT: ld.param.b64 %rd1, [local_unordered_volatile_cta_param_0];4048; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4049; CHECK-NEXT: ld.param.b64 %rd2, [local_unordered_volatile_cta_param_1];4050; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4051; CHECK-NEXT: ld.param.b64 %rd3, [local_unordered_volatile_cta_param_2];4052; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4053; CHECK-NEXT: ld.param.b64 %rd4, [local_unordered_volatile_cta_param_3];4054; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4055; CHECK-NEXT: ld.param.b64 %rd5, [local_unordered_volatile_cta_param_4];4056; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4057; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4058; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4059; CHECK-NEXT: add.s32 %r2, %r1, 1;4060; CHECK-NEXT: st.local.b32 [%rd3], %r2;4061; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4062; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4063; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4064; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4065; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4066; CHECK-NEXT: st.local.b32 [%rd5], %r4;4067; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4068; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4069; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4070; CHECK-NEXT: ret;4071 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("block") unordered, align 14072 %a.add = add i8 %a.load, 14073 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("block") unordered, align 14074 4075 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("block") unordered, align 24076 %b.add = add i16 %b.load, 14077 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("block") unordered, align 24078 4079 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("block") unordered, align 44080 %c.add = add i32 %c.load, 14081 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("block") unordered, align 44082 4083 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("block") unordered, align 84084 %d.add = add i64 %d.load, 14085 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("block") unordered, align 84086 4087 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("block") unordered, align 44088 %e.add = fadd float %e.load, 1.4089 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("block") unordered, align 44090 4091 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("block") unordered, align 84092 %f.add = fadd double %f.load, 1.4093 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("block") unordered, align 84094 4095 ret void4096}4097 4098; CHECK-LABEL: local_monotonic_gpu4099define void @local_monotonic_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4100; CHECK-LABEL: local_monotonic_gpu(4101; CHECK: {4102; CHECK-NEXT: .reg .b16 %rs<5>;4103; CHECK-NEXT: .reg .b32 %r<5>;4104; CHECK-NEXT: .reg .b64 %rd<10>;4105; CHECK-EMPTY:4106; CHECK-NEXT: // %bb.0:4107; CHECK-NEXT: ld.param.b64 %rd1, [local_monotonic_gpu_param_0];4108; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4109; CHECK-NEXT: ld.param.b64 %rd2, [local_monotonic_gpu_param_1];4110; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4111; CHECK-NEXT: ld.param.b64 %rd3, [local_monotonic_gpu_param_2];4112; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4113; CHECK-NEXT: ld.param.b64 %rd4, [local_monotonic_gpu_param_3];4114; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4115; CHECK-NEXT: ld.param.b64 %rd5, [local_monotonic_gpu_param_4];4116; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4117; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4118; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4119; CHECK-NEXT: add.s32 %r2, %r1, 1;4120; CHECK-NEXT: st.local.b32 [%rd3], %r2;4121; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4122; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4123; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4124; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4125; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4126; CHECK-NEXT: st.local.b32 [%rd5], %r4;4127; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4128; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4129; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4130; CHECK-NEXT: ret;4131 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("device") monotonic, align 14132 %a.add = add i8 %a.load, 14133 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("device") monotonic, align 14134 4135 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("device") monotonic, align 24136 %b.add = add i16 %b.load, 14137 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("device") monotonic, align 24138 4139 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("device") monotonic, align 44140 %c.add = add i32 %c.load, 14141 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("device") monotonic, align 44142 4143 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("device") monotonic, align 84144 %d.add = add i64 %d.load, 14145 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("device") monotonic, align 84146 4147 %e.load = load atomic float, ptr addrspace(5) %e syncscope("device") monotonic, align 44148 %e.add = fadd float %e.load, 1.4149 store atomic float %e.add, ptr addrspace(5) %e syncscope("device") monotonic, align 44150 4151 %f.load = load atomic double, ptr addrspace(5) %e syncscope("device") monotonic, align 84152 %f.add = fadd double %f.load, 1.4153 store atomic double %f.add, ptr addrspace(5) %e syncscope("device") monotonic, align 84154 4155 ret void4156}4157 4158; CHECK-LABEL: local_monotonic_volatile_gpu4159define void @local_monotonic_volatile_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4160; CHECK-LABEL: local_monotonic_volatile_gpu(4161; CHECK: {4162; CHECK-NEXT: .reg .b16 %rs<5>;4163; CHECK-NEXT: .reg .b32 %r<5>;4164; CHECK-NEXT: .reg .b64 %rd<10>;4165; CHECK-EMPTY:4166; CHECK-NEXT: // %bb.0:4167; CHECK-NEXT: ld.param.b64 %rd1, [local_monotonic_volatile_gpu_param_0];4168; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4169; CHECK-NEXT: ld.param.b64 %rd2, [local_monotonic_volatile_gpu_param_1];4170; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4171; CHECK-NEXT: ld.param.b64 %rd3, [local_monotonic_volatile_gpu_param_2];4172; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4173; CHECK-NEXT: ld.param.b64 %rd4, [local_monotonic_volatile_gpu_param_3];4174; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4175; CHECK-NEXT: ld.param.b64 %rd5, [local_monotonic_volatile_gpu_param_4];4176; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4177; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4178; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4179; CHECK-NEXT: add.s32 %r2, %r1, 1;4180; CHECK-NEXT: st.local.b32 [%rd3], %r2;4181; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4182; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4183; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4184; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4185; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4186; CHECK-NEXT: st.local.b32 [%rd5], %r4;4187; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4188; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4189; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4190; CHECK-NEXT: ret;4191 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("device") monotonic, align 14192 %a.add = add i8 %a.load, 14193 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("device") monotonic, align 14194 4195 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("device") monotonic, align 24196 %b.add = add i16 %b.load, 14197 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("device") monotonic, align 24198 4199 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("device") monotonic, align 44200 %c.add = add i32 %c.load, 14201 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("device") monotonic, align 44202 4203 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("device") monotonic, align 84204 %d.add = add i64 %d.load, 14205 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("device") monotonic, align 84206 4207 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("device") monotonic, align 44208 %e.add = fadd float %e.load, 1.4209 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("device") monotonic, align 44210 4211 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("device") monotonic, align 84212 %f.add = fadd double %f.load, 1.4213 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("device") monotonic, align 84214 4215 ret void4216}4217 4218; CHECK-LABEL: local_monotonic_cta4219define void @local_monotonic_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4220; CHECK-LABEL: local_monotonic_cta(4221; CHECK: {4222; CHECK-NEXT: .reg .b16 %rs<5>;4223; CHECK-NEXT: .reg .b32 %r<5>;4224; CHECK-NEXT: .reg .b64 %rd<10>;4225; CHECK-EMPTY:4226; CHECK-NEXT: // %bb.0:4227; CHECK-NEXT: ld.param.b64 %rd1, [local_monotonic_cta_param_0];4228; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4229; CHECK-NEXT: ld.param.b64 %rd2, [local_monotonic_cta_param_1];4230; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4231; CHECK-NEXT: ld.param.b64 %rd3, [local_monotonic_cta_param_2];4232; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4233; CHECK-NEXT: ld.param.b64 %rd4, [local_monotonic_cta_param_3];4234; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4235; CHECK-NEXT: ld.param.b64 %rd5, [local_monotonic_cta_param_4];4236; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4237; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4238; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4239; CHECK-NEXT: add.s32 %r2, %r1, 1;4240; CHECK-NEXT: st.local.b32 [%rd3], %r2;4241; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4242; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4243; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4244; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4245; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4246; CHECK-NEXT: st.local.b32 [%rd5], %r4;4247; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4248; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4249; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4250; CHECK-NEXT: ret;4251 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("block") monotonic, align 14252 %a.add = add i8 %a.load, 14253 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("block") monotonic, align 14254 4255 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("block") monotonic, align 24256 %b.add = add i16 %b.load, 14257 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("block") monotonic, align 24258 4259 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("block") monotonic, align 44260 %c.add = add i32 %c.load, 14261 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("block") monotonic, align 44262 4263 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("block") monotonic, align 84264 %d.add = add i64 %d.load, 14265 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("block") monotonic, align 84266 4267 %e.load = load atomic float, ptr addrspace(5) %e syncscope("block") monotonic, align 44268 %e.add = fadd float %e.load, 1.4269 store atomic float %e.add, ptr addrspace(5) %e syncscope("block") monotonic, align 44270 4271 %f.load = load atomic double, ptr addrspace(5) %e syncscope("block") monotonic, align 84272 %f.add = fadd double %f.load, 1.4273 store atomic double %f.add, ptr addrspace(5) %e syncscope("block") monotonic, align 84274 4275 ret void4276}4277 4278; CHECK-LABEL: local_monotonic_volatile_cta4279define void @local_monotonic_volatile_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4280; CHECK-LABEL: local_monotonic_volatile_cta(4281; CHECK: {4282; CHECK-NEXT: .reg .b16 %rs<5>;4283; CHECK-NEXT: .reg .b32 %r<5>;4284; CHECK-NEXT: .reg .b64 %rd<10>;4285; CHECK-EMPTY:4286; CHECK-NEXT: // %bb.0:4287; CHECK-NEXT: ld.param.b64 %rd1, [local_monotonic_volatile_cta_param_0];4288; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4289; CHECK-NEXT: ld.param.b64 %rd2, [local_monotonic_volatile_cta_param_1];4290; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4291; CHECK-NEXT: ld.param.b64 %rd3, [local_monotonic_volatile_cta_param_2];4292; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4293; CHECK-NEXT: ld.param.b64 %rd4, [local_monotonic_volatile_cta_param_3];4294; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4295; CHECK-NEXT: ld.param.b64 %rd5, [local_monotonic_volatile_cta_param_4];4296; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4297; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4298; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4299; CHECK-NEXT: add.s32 %r2, %r1, 1;4300; CHECK-NEXT: st.local.b32 [%rd3], %r2;4301; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4302; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4303; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4304; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4305; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4306; CHECK-NEXT: st.local.b32 [%rd5], %r4;4307; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4308; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4309; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4310; CHECK-NEXT: ret;4311 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("block") monotonic, align 14312 %a.add = add i8 %a.load, 14313 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("block") monotonic, align 14314 4315 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("block") monotonic, align 24316 %b.add = add i16 %b.load, 14317 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("block") monotonic, align 24318 4319 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("block") monotonic, align 44320 %c.add = add i32 %c.load, 14321 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("block") monotonic, align 44322 4323 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("block") monotonic, align 84324 %d.add = add i64 %d.load, 14325 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("block") monotonic, align 84326 4327 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("block") monotonic, align 44328 %e.add = fadd float %e.load, 1.4329 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("block") monotonic, align 44330 4331 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("block") monotonic, align 84332 %f.add = fadd double %f.load, 1.4333 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("block") monotonic, align 84334 4335 ret void4336}4337 4338; CHECK-LABEL: local_acq_rel_sys4339define void @local_acq_rel_sys(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4340; CHECK-LABEL: local_acq_rel_sys(4341; CHECK: {4342; CHECK-NEXT: .reg .b16 %rs<5>;4343; CHECK-NEXT: .reg .b32 %r<5>;4344; CHECK-NEXT: .reg .b64 %rd<10>;4345; CHECK-EMPTY:4346; CHECK-NEXT: // %bb.0:4347; CHECK-NEXT: ld.param.b64 %rd1, [local_acq_rel_sys_param_0];4348; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4349; CHECK-NEXT: ld.param.b64 %rd2, [local_acq_rel_sys_param_1];4350; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4351; CHECK-NEXT: ld.param.b64 %rd3, [local_acq_rel_sys_param_2];4352; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4353; CHECK-NEXT: ld.param.b64 %rd4, [local_acq_rel_sys_param_3];4354; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4355; CHECK-NEXT: ld.param.b64 %rd5, [local_acq_rel_sys_param_4];4356; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4357; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4358; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4359; CHECK-NEXT: add.s32 %r2, %r1, 1;4360; CHECK-NEXT: st.local.b32 [%rd3], %r2;4361; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4362; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4363; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4364; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4365; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4366; CHECK-NEXT: st.local.b32 [%rd5], %r4;4367; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4368; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4369; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4370; CHECK-NEXT: ret;4371 %a.load = load atomic i8, ptr addrspace(5) %a acquire, align 14372 %a.add = add i8 %a.load, 14373 store atomic i8 %a.add, ptr addrspace(5) %a release, align 14374 4375 %b.load = load atomic i16, ptr addrspace(5) %b acquire, align 24376 %b.add = add i16 %b.load, 14377 store atomic i16 %b.add, ptr addrspace(5) %b release, align 24378 4379 %c.load = load atomic i32, ptr addrspace(5) %c acquire, align 44380 %c.add = add i32 %c.load, 14381 store atomic i32 %c.add, ptr addrspace(5) %c release, align 44382 4383 %d.load = load atomic i64, ptr addrspace(5) %d acquire, align 84384 %d.add = add i64 %d.load, 14385 store atomic i64 %d.add, ptr addrspace(5) %d release, align 84386 4387 %e.load = load atomic float, ptr addrspace(5) %e acquire, align 44388 %e.add = fadd float %e.load, 1.4389 store atomic float %e.add, ptr addrspace(5) %e release, align 44390 4391 %f.load = load atomic double, ptr addrspace(5) %e acquire, align 84392 %f.add = fadd double %f.load, 1.4393 store atomic double %f.add, ptr addrspace(5) %e release, align 84394 4395 ret void4396}4397 4398; CHECK-LABEL: local_acq_rel_volatile_sys4399define void @local_acq_rel_volatile_sys(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4400; CHECK-LABEL: local_acq_rel_volatile_sys(4401; CHECK: {4402; CHECK-NEXT: .reg .b16 %rs<5>;4403; CHECK-NEXT: .reg .b32 %r<5>;4404; CHECK-NEXT: .reg .b64 %rd<10>;4405; CHECK-EMPTY:4406; CHECK-NEXT: // %bb.0:4407; CHECK-NEXT: ld.param.b64 %rd1, [local_acq_rel_volatile_sys_param_0];4408; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4409; CHECK-NEXT: ld.param.b64 %rd2, [local_acq_rel_volatile_sys_param_1];4410; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4411; CHECK-NEXT: ld.param.b64 %rd3, [local_acq_rel_volatile_sys_param_2];4412; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4413; CHECK-NEXT: ld.param.b64 %rd4, [local_acq_rel_volatile_sys_param_3];4414; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4415; CHECK-NEXT: ld.param.b64 %rd5, [local_acq_rel_volatile_sys_param_4];4416; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4417; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4418; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4419; CHECK-NEXT: add.s32 %r2, %r1, 1;4420; CHECK-NEXT: st.local.b32 [%rd3], %r2;4421; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4422; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4423; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4424; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4425; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4426; CHECK-NEXT: st.local.b32 [%rd5], %r4;4427; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4428; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4429; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4430; CHECK-NEXT: ret;4431 %a.load = load atomic volatile i8, ptr addrspace(5) %a acquire, align 14432 %a.add = add i8 %a.load, 14433 store atomic volatile i8 %a.add, ptr addrspace(5) %a release, align 14434 4435 %b.load = load atomic volatile i16, ptr addrspace(5) %b acquire, align 24436 %b.add = add i16 %b.load, 14437 store atomic volatile i16 %b.add, ptr addrspace(5) %b release, align 24438 4439 %c.load = load atomic volatile i32, ptr addrspace(5) %c acquire, align 44440 %c.add = add i32 %c.load, 14441 store atomic volatile i32 %c.add, ptr addrspace(5) %c release, align 44442 4443 %d.load = load atomic volatile i64, ptr addrspace(5) %d acquire, align 84444 %d.add = add i64 %d.load, 14445 store atomic volatile i64 %d.add, ptr addrspace(5) %d release, align 84446 4447 %e.load = load atomic volatile float, ptr addrspace(5) %e acquire, align 44448 %e.add = fadd float %e.load, 1.4449 store atomic volatile float %e.add, ptr addrspace(5) %e release, align 44450 4451 %f.load = load atomic volatile double, ptr addrspace(5) %e acquire, align 84452 %f.add = fadd double %f.load, 1.4453 store atomic volatile double %f.add, ptr addrspace(5) %e release, align 84454 4455 ret void4456}4457 4458; CHECK-LABEL: local_acq_rel_gpu4459define void @local_acq_rel_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4460; CHECK-LABEL: local_acq_rel_gpu(4461; CHECK: {4462; CHECK-NEXT: .reg .b16 %rs<5>;4463; CHECK-NEXT: .reg .b32 %r<5>;4464; CHECK-NEXT: .reg .b64 %rd<10>;4465; CHECK-EMPTY:4466; CHECK-NEXT: // %bb.0:4467; CHECK-NEXT: ld.param.b64 %rd1, [local_acq_rel_gpu_param_0];4468; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4469; CHECK-NEXT: ld.param.b64 %rd2, [local_acq_rel_gpu_param_1];4470; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4471; CHECK-NEXT: ld.param.b64 %rd3, [local_acq_rel_gpu_param_2];4472; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4473; CHECK-NEXT: ld.param.b64 %rd4, [local_acq_rel_gpu_param_3];4474; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4475; CHECK-NEXT: ld.param.b64 %rd5, [local_acq_rel_gpu_param_4];4476; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4477; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4478; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4479; CHECK-NEXT: add.s32 %r2, %r1, 1;4480; CHECK-NEXT: st.local.b32 [%rd3], %r2;4481; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4482; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4483; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4484; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4485; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4486; CHECK-NEXT: st.local.b32 [%rd5], %r4;4487; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4488; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4489; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4490; CHECK-NEXT: ret;4491 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("device") acquire, align 14492 %a.add = add i8 %a.load, 14493 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("device") release, align 14494 4495 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("device") acquire, align 24496 %b.add = add i16 %b.load, 14497 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("device") release, align 24498 4499 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("device") acquire, align 44500 %c.add = add i32 %c.load, 14501 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("device") release, align 44502 4503 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("device") acquire, align 84504 %d.add = add i64 %d.load, 14505 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("device") release, align 84506 4507 %e.load = load atomic float, ptr addrspace(5) %e syncscope("device") acquire, align 44508 %e.add = fadd float %e.load, 1.4509 store atomic float %e.add, ptr addrspace(5) %e syncscope("device") release, align 44510 4511 %f.load = load atomic double, ptr addrspace(5) %e syncscope("device") acquire, align 84512 %f.add = fadd double %f.load, 1.4513 store atomic double %f.add, ptr addrspace(5) %e syncscope("device") release, align 84514 4515 ret void4516}4517 4518; CHECK-LABEL: local_acq_rel_volatile_gpu4519define void @local_acq_rel_volatile_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4520; CHECK-LABEL: local_acq_rel_volatile_gpu(4521; CHECK: {4522; CHECK-NEXT: .reg .b16 %rs<5>;4523; CHECK-NEXT: .reg .b32 %r<5>;4524; CHECK-NEXT: .reg .b64 %rd<10>;4525; CHECK-EMPTY:4526; CHECK-NEXT: // %bb.0:4527; CHECK-NEXT: ld.param.b64 %rd1, [local_acq_rel_volatile_gpu_param_0];4528; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4529; CHECK-NEXT: ld.param.b64 %rd2, [local_acq_rel_volatile_gpu_param_1];4530; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4531; CHECK-NEXT: ld.param.b64 %rd3, [local_acq_rel_volatile_gpu_param_2];4532; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4533; CHECK-NEXT: ld.param.b64 %rd4, [local_acq_rel_volatile_gpu_param_3];4534; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4535; CHECK-NEXT: ld.param.b64 %rd5, [local_acq_rel_volatile_gpu_param_4];4536; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4537; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4538; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4539; CHECK-NEXT: add.s32 %r2, %r1, 1;4540; CHECK-NEXT: st.local.b32 [%rd3], %r2;4541; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4542; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4543; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4544; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4545; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4546; CHECK-NEXT: st.local.b32 [%rd5], %r4;4547; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4548; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4549; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4550; CHECK-NEXT: ret;4551 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("device") acquire, align 14552 %a.add = add i8 %a.load, 14553 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("device") release, align 14554 4555 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("device") acquire, align 24556 %b.add = add i16 %b.load, 14557 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("device") release, align 24558 4559 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("device") acquire, align 44560 %c.add = add i32 %c.load, 14561 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("device") release, align 44562 4563 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("device") acquire, align 84564 %d.add = add i64 %d.load, 14565 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("device") release, align 84566 4567 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("device") acquire, align 44568 %e.add = fadd float %e.load, 1.4569 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("device") release, align 44570 4571 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("device") acquire, align 84572 %f.add = fadd double %f.load, 1.4573 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("device") release, align 84574 4575 ret void4576}4577 4578; CHECK-LABEL: local_acq_rel_cta4579define void @local_acq_rel_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4580; CHECK-LABEL: local_acq_rel_cta(4581; CHECK: {4582; CHECK-NEXT: .reg .b16 %rs<5>;4583; CHECK-NEXT: .reg .b32 %r<5>;4584; CHECK-NEXT: .reg .b64 %rd<10>;4585; CHECK-EMPTY:4586; CHECK-NEXT: // %bb.0:4587; CHECK-NEXT: ld.param.b64 %rd1, [local_acq_rel_cta_param_0];4588; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4589; CHECK-NEXT: ld.param.b64 %rd2, [local_acq_rel_cta_param_1];4590; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4591; CHECK-NEXT: ld.param.b64 %rd3, [local_acq_rel_cta_param_2];4592; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4593; CHECK-NEXT: ld.param.b64 %rd4, [local_acq_rel_cta_param_3];4594; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4595; CHECK-NEXT: ld.param.b64 %rd5, [local_acq_rel_cta_param_4];4596; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4597; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4598; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4599; CHECK-NEXT: add.s32 %r2, %r1, 1;4600; CHECK-NEXT: st.local.b32 [%rd3], %r2;4601; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4602; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4603; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4604; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4605; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4606; CHECK-NEXT: st.local.b32 [%rd5], %r4;4607; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4608; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4609; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4610; CHECK-NEXT: ret;4611 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("block") acquire, align 14612 %a.add = add i8 %a.load, 14613 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("block") release, align 14614 4615 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("block") acquire, align 24616 %b.add = add i16 %b.load, 14617 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("block") release, align 24618 4619 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("block") acquire, align 44620 %c.add = add i32 %c.load, 14621 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("block") release, align 44622 4623 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("block") acquire, align 84624 %d.add = add i64 %d.load, 14625 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("block") release, align 84626 4627 %e.load = load atomic float, ptr addrspace(5) %e syncscope("block") acquire, align 44628 %e.add = fadd float %e.load, 1.4629 store atomic float %e.add, ptr addrspace(5) %e syncscope("block") release, align 44630 4631 %f.load = load atomic double, ptr addrspace(5) %e syncscope("block") acquire, align 84632 %f.add = fadd double %f.load, 1.4633 store atomic double %f.add, ptr addrspace(5) %e syncscope("block") release, align 84634 4635 ret void4636}4637 4638; CHECK-LABEL: local_acq_rel_volatile_cta4639define void @local_acq_rel_volatile_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4640; CHECK-LABEL: local_acq_rel_volatile_cta(4641; CHECK: {4642; CHECK-NEXT: .reg .b16 %rs<5>;4643; CHECK-NEXT: .reg .b32 %r<5>;4644; CHECK-NEXT: .reg .b64 %rd<10>;4645; CHECK-EMPTY:4646; CHECK-NEXT: // %bb.0:4647; CHECK-NEXT: ld.param.b64 %rd1, [local_acq_rel_volatile_cta_param_0];4648; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4649; CHECK-NEXT: ld.param.b64 %rd2, [local_acq_rel_volatile_cta_param_1];4650; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4651; CHECK-NEXT: ld.param.b64 %rd3, [local_acq_rel_volatile_cta_param_2];4652; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4653; CHECK-NEXT: ld.param.b64 %rd4, [local_acq_rel_volatile_cta_param_3];4654; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4655; CHECK-NEXT: ld.param.b64 %rd5, [local_acq_rel_volatile_cta_param_4];4656; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4657; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4658; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4659; CHECK-NEXT: add.s32 %r2, %r1, 1;4660; CHECK-NEXT: st.local.b32 [%rd3], %r2;4661; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4662; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4663; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4664; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4665; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4666; CHECK-NEXT: st.local.b32 [%rd5], %r4;4667; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4668; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4669; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4670; CHECK-NEXT: ret;4671 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("block") acquire, align 14672 %a.add = add i8 %a.load, 14673 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("block") release, align 14674 4675 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("block") acquire, align 24676 %b.add = add i16 %b.load, 14677 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("block") release, align 24678 4679 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("block") acquire, align 44680 %c.add = add i32 %c.load, 14681 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("block") release, align 44682 4683 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("block") acquire, align 84684 %d.add = add i64 %d.load, 14685 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("block") release, align 84686 4687 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("block") acquire, align 44688 %e.add = fadd float %e.load, 1.4689 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("block") release, align 44690 4691 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("block") acquire, align 84692 %f.add = fadd double %f.load, 1.4693 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("block") release, align 84694 4695 ret void4696}4697 4698; CHECK-LABEL: local_seq_cst_sys4699define void @local_seq_cst_sys(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4700; CHECK-LABEL: local_seq_cst_sys(4701; CHECK: {4702; CHECK-NEXT: .reg .b16 %rs<5>;4703; CHECK-NEXT: .reg .b32 %r<5>;4704; CHECK-NEXT: .reg .b64 %rd<10>;4705; CHECK-EMPTY:4706; CHECK-NEXT: // %bb.0:4707; CHECK-NEXT: ld.param.b64 %rd1, [local_seq_cst_sys_param_0];4708; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4709; CHECK-NEXT: ld.param.b64 %rd2, [local_seq_cst_sys_param_1];4710; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4711; CHECK-NEXT: ld.param.b64 %rd3, [local_seq_cst_sys_param_2];4712; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4713; CHECK-NEXT: ld.param.b64 %rd4, [local_seq_cst_sys_param_3];4714; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4715; CHECK-NEXT: ld.param.b64 %rd5, [local_seq_cst_sys_param_4];4716; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4717; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4718; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4719; CHECK-NEXT: add.s32 %r2, %r1, 1;4720; CHECK-NEXT: st.local.b32 [%rd3], %r2;4721; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4722; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4723; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4724; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4725; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4726; CHECK-NEXT: st.local.b32 [%rd5], %r4;4727; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4728; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4729; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4730; CHECK-NEXT: ret;4731 %a.load = load atomic i8, ptr addrspace(5) %a seq_cst, align 14732 %a.add = add i8 %a.load, 14733 store atomic i8 %a.add, ptr addrspace(5) %a seq_cst, align 14734 4735 %b.load = load atomic i16, ptr addrspace(5) %b seq_cst, align 24736 %b.add = add i16 %b.load, 14737 store atomic i16 %b.add, ptr addrspace(5) %b seq_cst, align 24738 4739 %c.load = load atomic i32, ptr addrspace(5) %c seq_cst, align 44740 %c.add = add i32 %c.load, 14741 store atomic i32 %c.add, ptr addrspace(5) %c seq_cst, align 44742 4743 %d.load = load atomic i64, ptr addrspace(5) %d seq_cst, align 84744 %d.add = add i64 %d.load, 14745 store atomic i64 %d.add, ptr addrspace(5) %d seq_cst, align 84746 4747 %e.load = load atomic float, ptr addrspace(5) %e seq_cst, align 44748 %e.add = fadd float %e.load, 1.4749 store atomic float %e.add, ptr addrspace(5) %e seq_cst, align 44750 4751 %f.load = load atomic double, ptr addrspace(5) %e seq_cst, align 84752 %f.add = fadd double %f.load, 1.4753 store atomic double %f.add, ptr addrspace(5) %e seq_cst, align 84754 4755 ret void4756}4757 4758; CHECK-LABEL: local_seq_cst_volatile_sys4759define void @local_seq_cst_volatile_sys(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4760; CHECK-LABEL: local_seq_cst_volatile_sys(4761; CHECK: {4762; CHECK-NEXT: .reg .b16 %rs<5>;4763; CHECK-NEXT: .reg .b32 %r<5>;4764; CHECK-NEXT: .reg .b64 %rd<10>;4765; CHECK-EMPTY:4766; CHECK-NEXT: // %bb.0:4767; CHECK-NEXT: ld.param.b64 %rd1, [local_seq_cst_volatile_sys_param_0];4768; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4769; CHECK-NEXT: ld.param.b64 %rd2, [local_seq_cst_volatile_sys_param_1];4770; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4771; CHECK-NEXT: ld.param.b64 %rd3, [local_seq_cst_volatile_sys_param_2];4772; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4773; CHECK-NEXT: ld.param.b64 %rd4, [local_seq_cst_volatile_sys_param_3];4774; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4775; CHECK-NEXT: ld.param.b64 %rd5, [local_seq_cst_volatile_sys_param_4];4776; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4777; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4778; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4779; CHECK-NEXT: add.s32 %r2, %r1, 1;4780; CHECK-NEXT: st.local.b32 [%rd3], %r2;4781; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4782; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4783; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4784; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4785; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4786; CHECK-NEXT: st.local.b32 [%rd5], %r4;4787; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4788; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4789; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4790; CHECK-NEXT: ret;4791 %a.load = load atomic volatile i8, ptr addrspace(5) %a seq_cst, align 14792 %a.add = add i8 %a.load, 14793 store atomic volatile i8 %a.add, ptr addrspace(5) %a seq_cst, align 14794 4795 %b.load = load atomic volatile i16, ptr addrspace(5) %b seq_cst, align 24796 %b.add = add i16 %b.load, 14797 store atomic volatile i16 %b.add, ptr addrspace(5) %b seq_cst, align 24798 4799 %c.load = load atomic volatile i32, ptr addrspace(5) %c seq_cst, align 44800 %c.add = add i32 %c.load, 14801 store atomic volatile i32 %c.add, ptr addrspace(5) %c seq_cst, align 44802 4803 %d.load = load atomic volatile i64, ptr addrspace(5) %d seq_cst, align 84804 %d.add = add i64 %d.load, 14805 store atomic volatile i64 %d.add, ptr addrspace(5) %d seq_cst, align 84806 4807 %e.load = load atomic volatile float, ptr addrspace(5) %e seq_cst, align 44808 %e.add = fadd float %e.load, 1.4809 store atomic volatile float %e.add, ptr addrspace(5) %e seq_cst, align 44810 4811 %f.load = load atomic volatile double, ptr addrspace(5) %e seq_cst, align 84812 %f.add = fadd double %f.load, 1.4813 store atomic volatile double %f.add, ptr addrspace(5) %e seq_cst, align 84814 4815 ret void4816}4817 4818; CHECK-LABEL: local_seq_cst_gpu4819define void @local_seq_cst_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4820; CHECK-LABEL: local_seq_cst_gpu(4821; CHECK: {4822; CHECK-NEXT: .reg .b16 %rs<5>;4823; CHECK-NEXT: .reg .b32 %r<5>;4824; CHECK-NEXT: .reg .b64 %rd<10>;4825; CHECK-EMPTY:4826; CHECK-NEXT: // %bb.0:4827; CHECK-NEXT: ld.param.b64 %rd1, [local_seq_cst_gpu_param_0];4828; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4829; CHECK-NEXT: ld.param.b64 %rd2, [local_seq_cst_gpu_param_1];4830; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4831; CHECK-NEXT: ld.param.b64 %rd3, [local_seq_cst_gpu_param_2];4832; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4833; CHECK-NEXT: ld.param.b64 %rd4, [local_seq_cst_gpu_param_3];4834; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4835; CHECK-NEXT: ld.param.b64 %rd5, [local_seq_cst_gpu_param_4];4836; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4837; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4838; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4839; CHECK-NEXT: add.s32 %r2, %r1, 1;4840; CHECK-NEXT: st.local.b32 [%rd3], %r2;4841; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4842; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4843; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4844; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4845; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4846; CHECK-NEXT: st.local.b32 [%rd5], %r4;4847; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4848; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4849; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4850; CHECK-NEXT: ret;4851 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("device") seq_cst, align 14852 %a.add = add i8 %a.load, 14853 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("device") seq_cst, align 14854 4855 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("device") seq_cst, align 24856 %b.add = add i16 %b.load, 14857 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("device") seq_cst, align 24858 4859 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("device") seq_cst, align 44860 %c.add = add i32 %c.load, 14861 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("device") seq_cst, align 44862 4863 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("device") seq_cst, align 84864 %d.add = add i64 %d.load, 14865 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("device") seq_cst, align 84866 4867 %e.load = load atomic float, ptr addrspace(5) %e syncscope("device") seq_cst, align 44868 %e.add = fadd float %e.load, 1.4869 store atomic float %e.add, ptr addrspace(5) %e syncscope("device") seq_cst, align 44870 4871 %f.load = load atomic double, ptr addrspace(5) %e syncscope("device") seq_cst, align 84872 %f.add = fadd double %f.load, 1.4873 store atomic double %f.add, ptr addrspace(5) %e syncscope("device") seq_cst, align 84874 4875 ret void4876}4877 4878; CHECK-LABEL: local_seq_cst_volatile_gpu4879define void @local_seq_cst_volatile_gpu(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4880; CHECK-LABEL: local_seq_cst_volatile_gpu(4881; CHECK: {4882; CHECK-NEXT: .reg .b16 %rs<5>;4883; CHECK-NEXT: .reg .b32 %r<5>;4884; CHECK-NEXT: .reg .b64 %rd<10>;4885; CHECK-EMPTY:4886; CHECK-NEXT: // %bb.0:4887; CHECK-NEXT: ld.param.b64 %rd1, [local_seq_cst_volatile_gpu_param_0];4888; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4889; CHECK-NEXT: ld.param.b64 %rd2, [local_seq_cst_volatile_gpu_param_1];4890; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4891; CHECK-NEXT: ld.param.b64 %rd3, [local_seq_cst_volatile_gpu_param_2];4892; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4893; CHECK-NEXT: ld.param.b64 %rd4, [local_seq_cst_volatile_gpu_param_3];4894; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4895; CHECK-NEXT: ld.param.b64 %rd5, [local_seq_cst_volatile_gpu_param_4];4896; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4897; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4898; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4899; CHECK-NEXT: add.s32 %r2, %r1, 1;4900; CHECK-NEXT: st.local.b32 [%rd3], %r2;4901; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4902; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4903; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4904; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4905; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4906; CHECK-NEXT: st.local.b32 [%rd5], %r4;4907; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4908; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4909; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4910; CHECK-NEXT: ret;4911 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("device") seq_cst, align 14912 %a.add = add i8 %a.load, 14913 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("device") seq_cst, align 14914 4915 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("device") seq_cst, align 24916 %b.add = add i16 %b.load, 14917 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("device") seq_cst, align 24918 4919 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("device") seq_cst, align 44920 %c.add = add i32 %c.load, 14921 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("device") seq_cst, align 44922 4923 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("device") seq_cst, align 84924 %d.add = add i64 %d.load, 14925 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("device") seq_cst, align 84926 4927 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("device") seq_cst, align 44928 %e.add = fadd float %e.load, 1.4929 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("device") seq_cst, align 44930 4931 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("device") seq_cst, align 84932 %f.add = fadd double %f.load, 1.4933 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("device") seq_cst, align 84934 4935 ret void4936}4937 4938; CHECK-LABEL: local_seq_cst_cta4939define void @local_seq_cst_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {4940; CHECK-LABEL: local_seq_cst_cta(4941; CHECK: {4942; CHECK-NEXT: .reg .b16 %rs<5>;4943; CHECK-NEXT: .reg .b32 %r<5>;4944; CHECK-NEXT: .reg .b64 %rd<10>;4945; CHECK-EMPTY:4946; CHECK-NEXT: // %bb.0:4947; CHECK-NEXT: ld.param.b64 %rd1, [local_seq_cst_cta_param_0];4948; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];4949; CHECK-NEXT: ld.param.b64 %rd2, [local_seq_cst_cta_param_1];4950; CHECK-NEXT: add.s16 %rs2, %rs1, 1;4951; CHECK-NEXT: ld.param.b64 %rd3, [local_seq_cst_cta_param_2];4952; CHECK-NEXT: st.local.b8 [%rd1], %rs2;4953; CHECK-NEXT: ld.param.b64 %rd4, [local_seq_cst_cta_param_3];4954; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];4955; CHECK-NEXT: ld.param.b64 %rd5, [local_seq_cst_cta_param_4];4956; CHECK-NEXT: add.s16 %rs4, %rs3, 1;4957; CHECK-NEXT: st.local.b16 [%rd2], %rs4;4958; CHECK-NEXT: ld.local.b32 %r1, [%rd3];4959; CHECK-NEXT: add.s32 %r2, %r1, 1;4960; CHECK-NEXT: st.local.b32 [%rd3], %r2;4961; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];4962; CHECK-NEXT: add.s64 %rd7, %rd6, 1;4963; CHECK-NEXT: st.local.b64 [%rd4], %rd7;4964; CHECK-NEXT: ld.local.b32 %r3, [%rd5];4965; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;4966; CHECK-NEXT: st.local.b32 [%rd5], %r4;4967; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];4968; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;4969; CHECK-NEXT: st.local.b64 [%rd5], %rd9;4970; CHECK-NEXT: ret;4971 %a.load = load atomic i8, ptr addrspace(5) %a syncscope("block") seq_cst, align 14972 %a.add = add i8 %a.load, 14973 store atomic i8 %a.add, ptr addrspace(5) %a syncscope("block") seq_cst, align 14974 4975 %b.load = load atomic i16, ptr addrspace(5) %b syncscope("block") seq_cst, align 24976 %b.add = add i16 %b.load, 14977 store atomic i16 %b.add, ptr addrspace(5) %b syncscope("block") seq_cst, align 24978 4979 %c.load = load atomic i32, ptr addrspace(5) %c syncscope("block") seq_cst, align 44980 %c.add = add i32 %c.load, 14981 store atomic i32 %c.add, ptr addrspace(5) %c syncscope("block") seq_cst, align 44982 4983 %d.load = load atomic i64, ptr addrspace(5) %d syncscope("block") seq_cst, align 84984 %d.add = add i64 %d.load, 14985 store atomic i64 %d.add, ptr addrspace(5) %d syncscope("block") seq_cst, align 84986 4987 %e.load = load atomic float, ptr addrspace(5) %e syncscope("block") seq_cst, align 44988 %e.add = fadd float %e.load, 1.4989 store atomic float %e.add, ptr addrspace(5) %e syncscope("block") seq_cst, align 44990 4991 %f.load = load atomic double, ptr addrspace(5) %e syncscope("block") seq_cst, align 84992 %f.add = fadd double %f.load, 1.4993 store atomic double %f.add, ptr addrspace(5) %e syncscope("block") seq_cst, align 84994 4995 ret void4996}4997 4998; CHECK-LABEL: local_seq_cst_volatile_cta4999define void @local_seq_cst_volatile_cta(ptr addrspace(5) %a, ptr addrspace(5) %b, ptr addrspace(5) %c, ptr addrspace(5) %d, ptr addrspace(5) %e) local_unnamed_addr {5000; CHECK-LABEL: local_seq_cst_volatile_cta(5001; CHECK: {5002; CHECK-NEXT: .reg .b16 %rs<5>;5003; CHECK-NEXT: .reg .b32 %r<5>;5004; CHECK-NEXT: .reg .b64 %rd<10>;5005; CHECK-EMPTY:5006; CHECK-NEXT: // %bb.0:5007; CHECK-NEXT: ld.param.b64 %rd1, [local_seq_cst_volatile_cta_param_0];5008; CHECK-NEXT: ld.local.b8 %rs1, [%rd1];5009; CHECK-NEXT: ld.param.b64 %rd2, [local_seq_cst_volatile_cta_param_1];5010; CHECK-NEXT: add.s16 %rs2, %rs1, 1;5011; CHECK-NEXT: ld.param.b64 %rd3, [local_seq_cst_volatile_cta_param_2];5012; CHECK-NEXT: st.local.b8 [%rd1], %rs2;5013; CHECK-NEXT: ld.param.b64 %rd4, [local_seq_cst_volatile_cta_param_3];5014; CHECK-NEXT: ld.local.b16 %rs3, [%rd2];5015; CHECK-NEXT: ld.param.b64 %rd5, [local_seq_cst_volatile_cta_param_4];5016; CHECK-NEXT: add.s16 %rs4, %rs3, 1;5017; CHECK-NEXT: st.local.b16 [%rd2], %rs4;5018; CHECK-NEXT: ld.local.b32 %r1, [%rd3];5019; CHECK-NEXT: add.s32 %r2, %r1, 1;5020; CHECK-NEXT: st.local.b32 [%rd3], %r2;5021; CHECK-NEXT: ld.local.b64 %rd6, [%rd4];5022; CHECK-NEXT: add.s64 %rd7, %rd6, 1;5023; CHECK-NEXT: st.local.b64 [%rd4], %rd7;5024; CHECK-NEXT: ld.local.b32 %r3, [%rd5];5025; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;5026; CHECK-NEXT: st.local.b32 [%rd5], %r4;5027; CHECK-NEXT: ld.local.b64 %rd8, [%rd5];5028; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;5029; CHECK-NEXT: st.local.b64 [%rd5], %rd9;5030; CHECK-NEXT: ret;5031 %a.load = load atomic volatile i8, ptr addrspace(5) %a syncscope("block") seq_cst, align 15032 %a.add = add i8 %a.load, 15033 store atomic volatile i8 %a.add, ptr addrspace(5) %a syncscope("block") seq_cst, align 15034 5035 %b.load = load atomic volatile i16, ptr addrspace(5) %b syncscope("block") seq_cst, align 25036 %b.add = add i16 %b.load, 15037 store atomic volatile i16 %b.add, ptr addrspace(5) %b syncscope("block") seq_cst, align 25038 5039 %c.load = load atomic volatile i32, ptr addrspace(5) %c syncscope("block") seq_cst, align 45040 %c.add = add i32 %c.load, 15041 store atomic volatile i32 %c.add, ptr addrspace(5) %c syncscope("block") seq_cst, align 45042 5043 %d.load = load atomic volatile i64, ptr addrspace(5) %d syncscope("block") seq_cst, align 85044 %d.add = add i64 %d.load, 15045 store atomic volatile i64 %d.add, ptr addrspace(5) %d syncscope("block") seq_cst, align 85046 5047 %e.load = load atomic volatile float, ptr addrspace(5) %e syncscope("block") seq_cst, align 45048 %e.add = fadd float %e.load, 1.5049 store atomic volatile float %e.add, ptr addrspace(5) %e syncscope("block") seq_cst, align 45050 5051 %f.load = load atomic volatile double, ptr addrspace(5) %e syncscope("block") seq_cst, align 85052 %f.add = fadd double %f.load, 1.5053 store atomic volatile double %f.add, ptr addrspace(5) %e syncscope("block") seq_cst, align 85054 5055 ret void5056}5057