687 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-new-sme-abi -aarch64-streaming-hazard-size=0 -mattr=+sve,+sme2 < %s | FileCheck %s3 4declare void @callee()5declare void @callee_sm() "aarch64_pstate_sm_enabled"6declare void @callee_farg(float)7declare float @callee_farg_fret(float)8 9; normal caller -> streaming callees10define void @test0(ptr %callee) nounwind {11; CHECK-LABEL: test0:12; CHECK: // %bb.0:13; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill14; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill15; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill16; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill17; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill18; CHECK-NEXT: smstart sm19; CHECK-NEXT: bl callee_sm20; CHECK-NEXT: bl callee_sm21; CHECK-NEXT: smstop sm22; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload23; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload24; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload25; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload26; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload27; CHECK-NEXT: ret28 call void @callee_sm()29 call void @callee_sm()30 ret void31}32 33; streaming caller -> normal callees34define void @test1() nounwind "aarch64_pstate_sm_enabled" {35; CHECK-LABEL: test1:36; CHECK: // %bb.0:37; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill38; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill39; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill40; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill41; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill42; CHECK-NEXT: smstop sm43; CHECK-NEXT: bl callee44; CHECK-NEXT: bl callee45; CHECK-NEXT: smstart sm46; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload47; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload48; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload49; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload50; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload51; CHECK-NEXT: ret52 call void @callee()53 call void @callee()54 ret void55}56 57; streaming-compatible caller -> normal callees58define void @test2() nounwind "aarch64_pstate_sm_compatible" {59; CHECK-LABEL: test2:60; CHECK: // %bb.0:61; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill62; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill63; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill64; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill65; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill66; CHECK-NEXT: mrs x19, SVCR67; CHECK-NEXT: tbz w19, #0, .LBB2_268; CHECK-NEXT: // %bb.1:69; CHECK-NEXT: smstop sm70; CHECK-NEXT: .LBB2_2:71; CHECK-NEXT: bl callee72; CHECK-NEXT: bl callee73; CHECK-NEXT: tbz w19, #0, .LBB2_474; CHECK-NEXT: // %bb.3:75; CHECK-NEXT: smstart sm76; CHECK-NEXT: .LBB2_4:77; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload78; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload79; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload80; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload81; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload82; CHECK-NEXT: ret83 call void @callee()84 call void @callee()85 ret void86}87 88; streaming-compatible caller -> mixed callees89define void @test3() nounwind "aarch64_pstate_sm_compatible" {90; CHECK-LABEL: test3:91; CHECK: // %bb.0:92; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill93; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill94; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill95; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill96; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill97; CHECK-NEXT: mrs x19, SVCR98; CHECK-NEXT: tbnz w19, #0, .LBB3_299; CHECK-NEXT: // %bb.1:100; CHECK-NEXT: smstart sm101; CHECK-NEXT: .LBB3_2:102; CHECK-NEXT: bl callee_sm103; CHECK-NEXT: tbnz w19, #0, .LBB3_4104; CHECK-NEXT: // %bb.3:105; CHECK-NEXT: smstop sm106; CHECK-NEXT: .LBB3_4:107; CHECK-NEXT: tbz w19, #0, .LBB3_6108; CHECK-NEXT: // %bb.5:109; CHECK-NEXT: smstop sm110; CHECK-NEXT: .LBB3_6:111; CHECK-NEXT: bl callee112; CHECK-NEXT: tbz w19, #0, .LBB3_8113; CHECK-NEXT: // %bb.7:114; CHECK-NEXT: smstart sm115; CHECK-NEXT: .LBB3_8:116; CHECK-NEXT: tbnz w19, #0, .LBB3_10117; CHECK-NEXT: // %bb.9:118; CHECK-NEXT: smstart sm119; CHECK-NEXT: .LBB3_10:120; CHECK-NEXT: bl callee_sm121; CHECK-NEXT: tbnz w19, #0, .LBB3_12122; CHECK-NEXT: // %bb.11:123; CHECK-NEXT: smstop sm124; CHECK-NEXT: .LBB3_12:125; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload126; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload127; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload128; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload129; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload130; CHECK-NEXT: ret131 call void @callee_sm()132 call void @callee()133 call void @callee_sm()134 ret void135}136 137; streaming caller -> normal callees (pass 0.0f)138define void @test4() nounwind "aarch64_pstate_sm_enabled" {139; CHECK-LABEL: test4:140; CHECK: // %bb.0:141; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill142; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill143; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill144; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill145; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill146; CHECK-NEXT: smstop sm147; CHECK-NEXT: fmov s0, wzr148; CHECK-NEXT: bl callee_farg149; CHECK-NEXT: fmov s0, wzr150; CHECK-NEXT: bl callee_farg151; CHECK-NEXT: smstart sm152; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload153; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload154; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload155; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload156; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload157; CHECK-NEXT: ret158 call void @callee_farg(float zeroinitializer)159 call void @callee_farg(float zeroinitializer)160 ret void161}162 163; streaming caller -> normal callees (pass fp arg)164define void @test5(float %f) nounwind "aarch64_pstate_sm_enabled" {165; CHECK-LABEL: test5:166; CHECK: // %bb.0:167; CHECK-NEXT: sub sp, sp, #96168; CHECK-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill169; CHECK-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill170; CHECK-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill171; CHECK-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill172; CHECK-NEXT: str x30, [sp, #80] // 8-byte Spill173; CHECK-NEXT: str s0, [sp, #12] // 4-byte Spill174; CHECK-NEXT: smstop sm175; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Reload176; CHECK-NEXT: bl callee_farg177; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Reload178; CHECK-NEXT: bl callee_farg179; CHECK-NEXT: smstart sm180; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload181; CHECK-NEXT: ldr x30, [sp, #80] // 8-byte Reload182; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload183; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload184; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload185; CHECK-NEXT: add sp, sp, #96186; CHECK-NEXT: ret187 call void @callee_farg(float %f)188 call void @callee_farg(float %f)189 ret void190}191 192define float @test6(float %f) nounwind "aarch64_pstate_sm_enabled" {193; CHECK-LABEL: test6:194; CHECK: // %bb.0:195; CHECK-NEXT: sub sp, sp, #96196; CHECK-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill197; CHECK-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill198; CHECK-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill199; CHECK-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill200; CHECK-NEXT: str x30, [sp, #80] // 8-byte Spill201; CHECK-NEXT: str s0, [sp, #12] // 4-byte Spill202; CHECK-NEXT: smstop sm203; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Reload204; CHECK-NEXT: bl callee_farg_fret205; CHECK-NEXT: bl callee_farg_fret206; CHECK-NEXT: str s0, [sp, #12] // 4-byte Spill207; CHECK-NEXT: smstart sm208; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload209; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Reload210; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload211; CHECK-NEXT: ldr x30, [sp, #80] // 8-byte Reload212; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload213; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload214; CHECK-NEXT: add sp, sp, #96215; CHECK-NEXT: ret216 %res0 = call float @callee_farg_fret(float %f)217 %res1 = call float @callee_farg_fret(float %res0)218 ret float %res1219}220 221; save/restore zt0 to stack is not yet optimised away by the pass,222; because of the ldr/str of zt0, which will need some further analysis223; to make sure if the redundant str can be removed.224define void @test7() nounwind "aarch64_inout_zt0" {225; CHECK-LABEL: test7:226; CHECK: // %bb.0:227; CHECK-NEXT: sub sp, sp, #80228; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill229; CHECK-NEXT: mov x19, sp230; CHECK-NEXT: str zt0, [x19]231; CHECK-NEXT: smstop za232; CHECK-NEXT: bl callee233; CHECK-NEXT: bl callee234; CHECK-NEXT: smstart za235; CHECK-NEXT: ldr zt0, [x19]236; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload237; CHECK-NEXT: add sp, sp, #80238; CHECK-NEXT: ret239 call void @callee()240 call void @callee()241 ret void242}243 244; test that 'smstop za' is not cancelled out with 'smstart sm'.245define void @test8() nounwind "aarch64_pstate_sm_enabled" {246; CHECK-LABEL: test8:247; CHECK: // %bb.0:248; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill249; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill250; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill251; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill252; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill253; CHECK-NEXT: smstop sm254; CHECK-NEXT: bl callee255; CHECK-NEXT: smstart sm256; CHECK-NEXT: smstop za257; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload258; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload259; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload260; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload261; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload262; CHECK-NEXT: ret263 call void @callee()264 call void @llvm.aarch64.sme.za.disable()265 ret void266}267 268; test that the 'smstart' and 'smstop' are entirely removed,269; along with any code to read 'vg' for the CFI.270define void @test9() "aarch64_pstate_sm_body" {271; CHECK-LABEL: test9:272; CHECK: // %bb.0:273; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill274; CHECK-NEXT: .cfi_def_cfa_offset 16275; CHECK-NEXT: .cfi_offset w30, -8276; CHECK-NEXT: .cfi_offset w29, -16277; CHECK-NEXT: bl callee278; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload279; CHECK-NEXT: ret280 call void @callee()281 ret void282}283 284; test that if the 'smstart' and 'smtop' are entirely removed in a locally285; streaming function, we use the FP, not an expression to describe the CFA.286define aarch64_sve_vector_pcs void @test9_1() "aarch64_pstate_sm_body" {287; CHECK-LABEL: test9_1:288; CHECK: // %bb.0:289; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill290; CHECK-NEXT: str x28, [sp, #16] // 8-byte Spill291; CHECK-NEXT: mov x29, sp292; CHECK-NEXT: addsvl sp, sp, #-18293; CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Spill294; CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Spill295; CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Spill296; CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Spill297; CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Spill298; CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Spill299; CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Spill300; CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Spill301; CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Spill302; CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Spill303; CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Spill304; CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Spill305; CHECK-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill306; CHECK-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill307; CHECK-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill308; CHECK-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill309; CHECK-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill310; CHECK-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill311; CHECK-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill312; CHECK-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill313; CHECK-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill314; CHECK-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill315; CHECK-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill316; CHECK-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill317; CHECK-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill318; CHECK-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill319; CHECK-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill320; CHECK-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill321; CHECK-NEXT: .cfi_def_cfa w29, 32322; CHECK-NEXT: .cfi_offset w28, -16323; CHECK-NEXT: .cfi_offset w30, -24324; CHECK-NEXT: .cfi_offset w29, -32325; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * VG - 32326; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * VG - 32327; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * VG - 32328; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * VG - 32329; CHECK-NEXT: .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * VG - 32330; CHECK-NEXT: .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * VG - 32331; CHECK-NEXT: .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * VG - 32332; CHECK-NEXT: .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * VG - 32333; CHECK-NEXT: bl callee334; CHECK-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload335; CHECK-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload336; CHECK-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload337; CHECK-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload338; CHECK-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload339; CHECK-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload340; CHECK-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload341; CHECK-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload342; CHECK-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload343; CHECK-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload344; CHECK-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload345; CHECK-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload346; CHECK-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload347; CHECK-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload348; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload349; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload350; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload351; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Reload352; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Reload353; CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Reload354; CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Reload355; CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Reload356; CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Reload357; CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Reload358; CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Reload359; CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Reload360; CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Reload361; CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Reload362; CHECK-NEXT: addsvl sp, sp, #18363; CHECK-NEXT: ldr x28, [sp, #16] // 8-byte Reload364; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload365; CHECK-NEXT: ret366 call void @callee()367 ret void368}369 370; similar to above, but in this case only the first371; 'smstart, smstop' pair can be removed and the code required372; for the CFI is still needed.373define void @test10() "aarch64_pstate_sm_body" {374; CHECK-LABEL: test10:375; CHECK: // %bb.0:376; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill377; CHECK-NEXT: .cfi_def_cfa_offset 96378; CHECK-NEXT: cntd x9379; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill380; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill381; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill382; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill383; CHECK-NEXT: str x9, [sp, #80] // 8-byte Spill384; CHECK-NEXT: .cfi_offset vg, -16385; CHECK-NEXT: .cfi_offset w30, -24386; CHECK-NEXT: .cfi_offset w29, -32387; CHECK-NEXT: .cfi_offset b8, -40388; CHECK-NEXT: .cfi_offset b9, -48389; CHECK-NEXT: .cfi_offset b10, -56390; CHECK-NEXT: .cfi_offset b11, -64391; CHECK-NEXT: .cfi_offset b12, -72392; CHECK-NEXT: .cfi_offset b13, -80393; CHECK-NEXT: .cfi_offset b14, -88394; CHECK-NEXT: .cfi_offset b15, -96395; CHECK-NEXT: bl callee396; CHECK-NEXT: smstart sm397; CHECK-NEXT: bl callee_sm398; CHECK-NEXT: smstop sm399; CHECK-NEXT: bl callee400; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload401; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload402; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload403; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload404; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload405; CHECK-NEXT: .cfi_def_cfa_offset 0406; CHECK-NEXT: .cfi_restore vg407; CHECK-NEXT: .cfi_restore w30408; CHECK-NEXT: .cfi_restore w29409; CHECK-NEXT: .cfi_restore b8410; CHECK-NEXT: .cfi_restore b9411; CHECK-NEXT: .cfi_restore b10412; CHECK-NEXT: .cfi_restore b11413; CHECK-NEXT: .cfi_restore b12414; CHECK-NEXT: .cfi_restore b13415; CHECK-NEXT: .cfi_restore b14416; CHECK-NEXT: .cfi_restore b15417; CHECK-NEXT: ret418 call void @callee()419 call void @callee_sm()420 call void @callee()421 ret void422}423 424; test that an operation like a store is executed in the right425; streaming mode and blocks the optimization.426define void @test11(ptr %p) nounwind "aarch64_pstate_sm_enabled" {427; CHECK-LABEL: test11:428; CHECK: // %bb.0:429; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill430; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill431; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill432; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill433; CHECK-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill434; CHECK-NEXT: mov x19, x0435; CHECK-NEXT: smstop sm436; CHECK-NEXT: bl callee437; CHECK-NEXT: smstart sm438; CHECK-NEXT: mov z0.b, #0 // =0x0439; CHECK-NEXT: str z0, [x19]440; CHECK-NEXT: smstop sm441; CHECK-NEXT: bl callee442; CHECK-NEXT: smstart sm443; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload444; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload445; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload446; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload447; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload448; CHECK-NEXT: ret449 call void @callee()450 store <vscale x 16 x i8> zeroinitializer, ptr %p451 call void @callee()452 ret void453}454 455; test that 'smstart sm' and 'smstop za' don't get folded away together.456; we can further optimize this test by considering streaming mode457; separately from ZA.458define void @test12() "aarch64_pstate_sm_body" {459; CHECK-LABEL: test12:460; CHECK: // %bb.0:461; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill462; CHECK-NEXT: .cfi_def_cfa_offset 96463; CHECK-NEXT: cntd x9464; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill465; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill466; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill467; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill468; CHECK-NEXT: str x9, [sp, #80] // 8-byte Spill469; CHECK-NEXT: .cfi_offset vg, -16470; CHECK-NEXT: .cfi_offset w30, -24471; CHECK-NEXT: .cfi_offset w29, -32472; CHECK-NEXT: .cfi_offset b8, -40473; CHECK-NEXT: .cfi_offset b9, -48474; CHECK-NEXT: .cfi_offset b10, -56475; CHECK-NEXT: .cfi_offset b11, -64476; CHECK-NEXT: .cfi_offset b12, -72477; CHECK-NEXT: .cfi_offset b13, -80478; CHECK-NEXT: .cfi_offset b14, -88479; CHECK-NEXT: .cfi_offset b15, -96480; CHECK-NEXT: smstart sm481; CHECK-NEXT: smstop za482; CHECK-NEXT: smstop sm483; CHECK-NEXT: bl callee484; CHECK-NEXT: smstart sm485; CHECK-NEXT: smstart za486; CHECK-NEXT: smstop sm487; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload488; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload489; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload490; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload491; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload492; CHECK-NEXT: .cfi_def_cfa_offset 0493; CHECK-NEXT: .cfi_restore vg494; CHECK-NEXT: .cfi_restore w30495; CHECK-NEXT: .cfi_restore w29496; CHECK-NEXT: .cfi_restore b8497; CHECK-NEXT: .cfi_restore b9498; CHECK-NEXT: .cfi_restore b10499; CHECK-NEXT: .cfi_restore b11500; CHECK-NEXT: .cfi_restore b12501; CHECK-NEXT: .cfi_restore b13502; CHECK-NEXT: .cfi_restore b14503; CHECK-NEXT: .cfi_restore b15504; CHECK-NEXT: ret505 call void @llvm.aarch64.sme.za.disable()506 call void @callee()507 call void @llvm.aarch64.sme.za.enable()508 ret void509}510 511; We conservatively don't remove the smstart/smstop pair yet when there are COPY512; instructions that copy SVE registers, because we can't yet conclusively prove513; that this is safe (although for this example, it would be).514define void @test13(ptr %ptr) nounwind "aarch64_pstate_sm_enabled" {515; CHECK-LABEL: test13:516; CHECK: // %bb.0:517; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill518; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill519; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill520; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill521; CHECK-NEXT: str x29, [sp, #64] // 8-byte Spill522; CHECK-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill523; CHECK-NEXT: addvl sp, sp, #-1524; CHECK-NEXT: mov z0.s, #0 // =0x0525; CHECK-NEXT: str z0, [sp] // 16-byte Folded Spill526; CHECK-NEXT: smstop sm527; CHECK-NEXT: rdvl x8, #1528; CHECK-NEXT: addsvl x8, x8, #-1529; CHECK-NEXT: cbnz x8, .LBB14_2530; CHECK-NEXT: // %bb.1:531; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload532; CHECK-NEXT: mov x19, x0533; CHECK-NEXT: bl callee_farg_fret534; CHECK-NEXT: str z0, [sp] // 16-byte Folded Spill535; CHECK-NEXT: smstart sm536; CHECK-NEXT: smstop sm537; CHECK-NEXT: rdvl x8, #1538; CHECK-NEXT: addsvl x8, x8, #-1539; CHECK-NEXT: cbz x8, .LBB14_3540; CHECK-NEXT: .LBB14_2:541; CHECK-NEXT: brk #0x1542; CHECK-NEXT: .LBB14_3:543; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload544; CHECK-NEXT: bl callee_farg_fret545; CHECK-NEXT: str z0, [sp] // 16-byte Folded Spill546; CHECK-NEXT: smstart sm547; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload548; CHECK-NEXT: str z0, [x19]549; CHECK-NEXT: addvl sp, sp, #1550; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload551; CHECK-NEXT: ldr x29, [sp, #64] // 8-byte Reload552; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload553; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload554; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload555; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload556; CHECK-NEXT: ret557 %res0 = call <vscale x 4 x float> @callee_farg_fret(<vscale x 4 x float> zeroinitializer)558 %res1 = call <vscale x 4 x float> @callee_farg_fret(<vscale x 4 x float> %res0)559 store <vscale x 4 x float> %res1, ptr %ptr560 ret void561}562 563; normal caller -> streaming callees (with ZA state)564define void @test14(ptr %callee) nounwind "aarch64_inout_za" {565; CHECK-LABEL: test14:566; CHECK: // %bb.0:567; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill568; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill569; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill570; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill571; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill572; CHECK-NEXT: add x29, sp, #64573; CHECK-NEXT: str x19, [sp, #80] // 8-byte Spill574; CHECK-NEXT: sub sp, sp, #16575; CHECK-NEXT: rdsvl x8, #1576; CHECK-NEXT: mov x9, sp577; CHECK-NEXT: msub x9, x8, x8, x9578; CHECK-NEXT: mov sp, x9579; CHECK-NEXT: sub x10, x29, #80580; CHECK-NEXT: stp x9, x8, [x29, #-80]581; CHECK-NEXT: msr TPIDR2_EL0, x10582; CHECK-NEXT: smstart sm583; CHECK-NEXT: bl callee_sm584; CHECK-NEXT: bl callee_sm585; CHECK-NEXT: smstop sm586; CHECK-NEXT: smstart za587; CHECK-NEXT: mrs x8, TPIDR2_EL0588; CHECK-NEXT: sub x0, x29, #80589; CHECK-NEXT: cbnz x8, .LBB15_2590; CHECK-NEXT: // %bb.1:591; CHECK-NEXT: bl __arm_tpidr2_restore592; CHECK-NEXT: .LBB15_2:593; CHECK-NEXT: msr TPIDR2_EL0, xzr594; CHECK-NEXT: sub sp, x29, #64595; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload596; CHECK-NEXT: ldr x19, [sp, #80] // 8-byte Reload597; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload598; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload599; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload600; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload601; CHECK-NEXT: ret602 call void @callee_sm()603 call void @callee_sm()604 ret void605}606 607; normal caller -> streaming callees (with ZA agnostic state)608define void @test15(ptr %callee) nounwind "aarch64_za_state_agnostic" {609; CHECK-LABEL: test15:610; CHECK: // %bb.0:611; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill612; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill613; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill614; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill615; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill616; CHECK-NEXT: add x29, sp, #64617; CHECK-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill618; CHECK-NEXT: bl __arm_sme_state_size619; CHECK-NEXT: sub sp, sp, x0620; CHECK-NEXT: mov x20, sp621; CHECK-NEXT: mov x0, x20622; CHECK-NEXT: bl __arm_sme_save623; CHECK-NEXT: smstart sm624; CHECK-NEXT: bl callee_sm625; CHECK-NEXT: bl callee_sm626; CHECK-NEXT: smstop sm627; CHECK-NEXT: mov x0, x20628; CHECK-NEXT: bl __arm_sme_restore629; CHECK-NEXT: sub sp, x29, #64630; CHECK-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload631; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload632; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload633; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload634; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload635; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload636; CHECK-NEXT: ret637 call void @callee_sm()638 call void @callee_sm()639 ret void640}641 642; locally streaming caller -> normal callees (with ZA state)643define void @test16(ptr %callee) nounwind "aarch64_pstate_sm_body" "aarch64_new_za" {644; CHECK-LABEL: test16:645; CHECK: // %bb.0:646; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill647; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill648; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill649; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill650; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill651; CHECK-NEXT: add x29, sp, #64652; CHECK-NEXT: str x19, [sp, #80] // 8-byte Spill653; CHECK-NEXT: sub sp, sp, #16654; CHECK-NEXT: rdsvl x8, #1655; CHECK-NEXT: mov x9, sp656; CHECK-NEXT: msub x9, x8, x8, x9657; CHECK-NEXT: mov sp, x9658; CHECK-NEXT: stp x9, x8, [x29, #-80]659; CHECK-NEXT: mrs x8, TPIDR2_EL0660; CHECK-NEXT: cbz x8, .LBB17_2661; CHECK-NEXT: // %bb.1:662; CHECK-NEXT: bl __arm_tpidr2_save663; CHECK-NEXT: msr TPIDR2_EL0, xzr664; CHECK-NEXT: zero {za}665; CHECK-NEXT: .LBB17_2:666; CHECK-NEXT: smstart za667; CHECK-NEXT: smstart sm668; CHECK-NEXT: sub x8, x29, #80669; CHECK-NEXT: msr TPIDR2_EL0, x8670; CHECK-NEXT: smstop sm671; CHECK-NEXT: bl callee672; CHECK-NEXT: bl callee673; CHECK-NEXT: msr TPIDR2_EL0, xzr674; CHECK-NEXT: smstop za675; CHECK-NEXT: sub sp, x29, #64676; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload677; CHECK-NEXT: ldr x19, [sp, #80] // 8-byte Reload678; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload679; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload680; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload681; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload682; CHECK-NEXT: ret683 call void @callee()684 call void @callee()685 ret void686}687