304 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme -start-after=simplifycfg -enable-tail-merge=false -verify-machineinstrs < %s | FileCheck %s3 4declare void @normal_callee();5declare void @streaming_callee() "aarch64_pstate_sm_enabled";6declare void @streaming_compatible_callee() "aarch64_pstate_sm_compatible";7 8define void @locally_streaming_caller_streaming_callee() "aarch64_pstate_sm_body" nounwind {9; CHECK-LABEL: locally_streaming_caller_streaming_callee:10; CHECK: // %bb.0:11; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill12; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill13; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill14; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill15; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill16; CHECK-NEXT: smstart sm17; CHECK-NEXT: bl streaming_compatible_callee18; CHECK-NEXT: bl streaming_compatible_callee19; CHECK-NEXT: smstop sm20; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload21; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload22; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload23; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload24; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload25; CHECK-NEXT: ret26 27 call void @streaming_compatible_callee();28 call void @streaming_compatible_callee();29 ret void;30}31 32; Test that a streaming body and streaming interface, no smstart/smstop are emitted,33; because the function already is in streaming mode upon entry.34define void @streaming_and_locally_streaming_caller_streaming_callee() "aarch64_pstate_sm_enabled" "aarch64_pstate_sm_body" nounwind {35; CHECK-LABEL: streaming_and_locally_streaming_caller_streaming_callee:36; CHECK: // %bb.0:37; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill38; CHECK-NEXT: bl streaming_callee39; CHECK-NEXT: bl streaming_callee40; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload41; CHECK-NEXT: ret42 call void @streaming_callee();43 call void @streaming_callee();44 ret void;45}46 47define void @locally_streaming_multiple_exit(i64 %cond) "aarch64_pstate_sm_body" nounwind {48; CHECK-LABEL: locally_streaming_multiple_exit:49; CHECK: // %bb.0: // %entry50; CHECK-NEXT: stp d15, d14, [sp, #-64]! // 16-byte Folded Spill51; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill52; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill53; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill54; CHECK-NEXT: smstart sm55; CHECK-NEXT: cmp x0, #156; CHECK-NEXT: b.ne .LBB2_257; CHECK-NEXT: // %bb.1: // %if.else58; CHECK-NEXT: smstop sm59; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload60; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload61; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload62; CHECK-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload63; CHECK-NEXT: ret64; CHECK-NEXT: .LBB2_2: // %if.end65; CHECK-NEXT: smstop sm66; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload67; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload68; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload69; CHECK-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload70; CHECK-NEXT: ret71 72entry:73 %tobool = icmp eq i64 %cond, 174 br i1 %tobool, label %if.else, label %if.end75 76if.else:77 ret void;78 79if.end:80 ret void;81}82 83; Do a fixed-width vector add on a NEON vector.84; This tests that:85; * Incoming vector in v0.d isn't clobbered by the change in streaming mode.86; * Result vector is correctly preserved after smstop.87define <2 x i64> @locally_streaming_caller_no_callee(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind {88; CHECK-LABEL: locally_streaming_caller_no_callee:89; CHECK: // %bb.0:90; CHECK-NEXT: sub sp, sp, #8091; CHECK-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill92; CHECK-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill93; CHECK-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill94; CHECK-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill95; CHECK-NEXT: str q0, [sp] // 16-byte Spill96; CHECK-NEXT: smstart sm97; CHECK-NEXT: index z0.d, #0, #198; CHECK-NEXT: ldr q1, [sp] // 16-byte Reload99; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1100; CHECK-NEXT: add z0.d, z0.d, z1.d101; CHECK-NEXT: add z0.d, z0.d, #41 // =0x29102; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0103; CHECK-NEXT: str q0, [sp] // 16-byte Spill104; CHECK-NEXT: smstop sm105; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload106; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload107; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload108; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload109; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload110; CHECK-NEXT: add sp, sp, #80111; CHECK-NEXT: ret112 113 %add = add <2 x i64> %a, <i64 41, i64 42>;114 ret <2 x i64> %add;115}116 117; Test that we use the interface (not the function's body) to determine what118; streaming-mode to enter the callee. In this case the interface is normal, so119; pstate.sm must be 0 on entry and is 0 upon return from the callee.120define void @locally_streaming_caller_locally_streaming_callee() "aarch64_pstate_sm_body" nounwind {121; CHECK-LABEL: locally_streaming_caller_locally_streaming_callee:122; CHECK: // %bb.0:123; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill124; CHECK-NEXT: bl locally_streaming_caller_streaming_callee125; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload126; CHECK-NEXT: ret127 128 call void @locally_streaming_caller_streaming_callee();129 ret void;130}131 132;133; Test that a locally streaming function correctly retains the134; argument/result registers, because smstart/smstop instructions that are135; inserted to implement the arm_locally_streaming attribute thrashes the136; vector register contents.137;138 139define <2 x i64> @locally_streaming_caller_compatible_callee_vec_args_ret(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind {140; CHECK-LABEL: locally_streaming_caller_compatible_callee_vec_args_ret:141; CHECK: // %bb.0:142; CHECK-NEXT: sub sp, sp, #96143; CHECK-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill144; CHECK-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill145; CHECK-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill146; CHECK-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill147; CHECK-NEXT: str x30, [sp, #80] // 8-byte Spill148; CHECK-NEXT: str q0, [sp] // 16-byte Spill149; CHECK-NEXT: smstart sm150; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload151; CHECK-NEXT: bl streaming_compatible_callee_vec_args_ret152; CHECK-NEXT: str q0, [sp] // 16-byte Spill153; CHECK-NEXT: smstop sm154; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload155; CHECK-NEXT: ldr q0, [sp] // 16-byte Reload156; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload157; CHECK-NEXT: ldr x30, [sp, #80] // 8-byte Reload158; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload159; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload160; CHECK-NEXT: add sp, sp, #96161; CHECK-NEXT: ret162 %res = call <2 x i64> @streaming_compatible_callee_vec_args_ret(<2 x i64> %a) "aarch64_pstate_sm_compatible"163 ret <2 x i64> %res;164}165 166declare <2 x i64> @streaming_compatible_callee_vec_args_ret(<2 x i64>) "aarch64_pstate_sm_compatible"167 168define {<2 x i64>, <2 x i64>} @locally_streaming_caller_compatible_callee_struct_arg_ret({<2 x i64>, <2 x i64>} %arg) "aarch64_pstate_sm_body" nounwind {169; CHECK-LABEL: locally_streaming_caller_compatible_callee_struct_arg_ret:170; CHECK: // %bb.0:171; CHECK-NEXT: sub sp, sp, #112172; CHECK-NEXT: stp d15, d14, [sp, #32] // 16-byte Folded Spill173; CHECK-NEXT: stp d13, d12, [sp, #48] // 16-byte Folded Spill174; CHECK-NEXT: stp d11, d10, [sp, #64] // 16-byte Folded Spill175; CHECK-NEXT: stp d9, d8, [sp, #80] // 16-byte Folded Spill176; CHECK-NEXT: str x30, [sp, #96] // 8-byte Spill177; CHECK-NEXT: str q1, [sp, #16] // 16-byte Spill178; CHECK-NEXT: smstart sm179; CHECK-NEXT: ldr q0, [sp, #16] // 16-byte Reload180; CHECK-NEXT: bl streaming_compatible_callee_vec_arg_struct_ret181; CHECK-NEXT: stp q1, q0, [sp] // 32-byte Folded Spill182; CHECK-NEXT: smstop sm183; CHECK-NEXT: ldp q1, q0, [sp] // 32-byte Folded Reload184; CHECK-NEXT: ldr x30, [sp, #96] // 8-byte Reload185; CHECK-NEXT: ldp d9, d8, [sp, #80] // 16-byte Folded Reload186; CHECK-NEXT: ldp d11, d10, [sp, #64] // 16-byte Folded Reload187; CHECK-NEXT: ldp d13, d12, [sp, #48] // 16-byte Folded Reload188; CHECK-NEXT: ldp d15, d14, [sp, #32] // 16-byte Folded Reload189; CHECK-NEXT: add sp, sp, #112190; CHECK-NEXT: ret191 %v1.arg = extractvalue {<2 x i64>, <2 x i64>} %arg, 1192 %res = call {<2 x i64>, <2 x i64>} @streaming_compatible_callee_vec_arg_struct_ret(<2 x i64> %v1.arg) "aarch64_pstate_sm_compatible"193 ret {<2 x i64>, <2 x i64>} %res;194}195 196declare {<2 x i64>, <2 x i64>} @streaming_compatible_callee_vec_arg_struct_ret(<2 x i64>) "aarch64_pstate_sm_compatible"197 198; Test that we use `addsvl` for allocating any stack space for locals before `smstart`,199; such that the correct amount of stack space is allocated.200define void @locally_streaming_caller_alloca() nounwind "aarch64_pstate_sm_body" {201; CHECK-LABEL: locally_streaming_caller_alloca:202; CHECK: // %bb.0:203; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill204; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill205; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill206; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill207; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill208; CHECK-NEXT: addsvl sp, sp, #-1209; CHECK-NEXT: smstart sm210; CHECK-NEXT: mov x0, sp211; CHECK-NEXT: bl use_ptr212; CHECK-NEXT: smstop sm213; CHECK-NEXT: addsvl sp, sp, #1214; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload215; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload216; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload217; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload218; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload219; CHECK-NEXT: ret220 %alloca = alloca <vscale x 4 x i32>221 call void @use_ptr(ptr %alloca) "aarch64_pstate_sm_compatible"222 ret void223}224 225declare void @use_ptr(ptr) "aarch64_pstate_sm_compatible"226 227define double @call_to_intrinsic_without_chain(double %x) nounwind "aarch64_pstate_sm_body" {228; CHECK-LABEL: call_to_intrinsic_without_chain:229; CHECK: // %bb.0: // %entry230; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill231; CHECK-NEXT: bl cos232; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload233; CHECK-NEXT: ret234entry:235 %0 = call fast double @llvm.cos.f64(double %x)236 ret double %0237}238 239declare double @llvm.cos.f64(double)240 241 242define float @test_arg_survives_loop(float %arg, i32 %N) nounwind "aarch64_pstate_sm_body" {243; CHECK-LABEL: test_arg_survives_loop:244; CHECK: // %bb.0: // %entry245; CHECK-NEXT: sub sp, sp, #80246; CHECK-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill247; CHECK-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill248; CHECK-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill249; CHECK-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill250; CHECK-NEXT: str s0, [sp, #12] // 4-byte Spill251; CHECK-NEXT: smstart sm252; CHECK-NEXT: .LBB9_1: // %for.body253; CHECK-NEXT: // =>This Inner Loop Header: Depth=1254; CHECK-NEXT: subs w0, w0, #1255; CHECK-NEXT: b.ne .LBB9_1256; CHECK-NEXT: // %bb.2: // %for.cond.cleanup257; CHECK-NEXT: fmov s0, #1.00000000258; CHECK-NEXT: ldr s1, [sp, #12] // 4-byte Reload259; CHECK-NEXT: fadd s0, s1, s0260; CHECK-NEXT: str s0, [sp, #12] // 4-byte Spill261; CHECK-NEXT: smstop sm262; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload263; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Reload264; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload265; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload266; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload267; CHECK-NEXT: add sp, sp, #80268; CHECK-NEXT: ret269entry:270 br label %for.body271 272for.body:273 %i.02 = phi i32 [ 0, %entry ], [ %inc, %for.body ]274 %inc = add nuw nsw i32 %i.02, 1275 %exitcond.not = icmp eq i32 %inc, %N276 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body277 278for.cond.cleanup:279 %add = fadd float %arg, 1.000000e+00280 ret float %add281 282}283 284define void @disable_tailcallopt() "aarch64_pstate_sm_body" nounwind {285; CHECK-LABEL: disable_tailcallopt:286; CHECK: // %bb.0:287; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill288; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill289; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill290; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill291; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill292; CHECK-NEXT: smstart sm293; CHECK-NEXT: bl streaming_compatible_callee294; CHECK-NEXT: smstop sm295; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload296; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload297; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload298; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload299; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload300; CHECK-NEXT: ret301 tail call void @streaming_compatible_callee();302 ret void;303}304