644 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64 -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK3; RUN: llc -mtriple=aarch64 -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme -aarch64-new-sme-abi < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-NEWLOWERING4 5declare void @private_za_callee()6declare void @shared_za_callee() "aarch64_inout_za"7declare void @preserves_za_callee() "aarch64_preserves_za"8 9declare float @llvm.cos.f32(float)10 11; Test lazy-save mechanism for a single callee.12define void @test_lazy_save_1_callee() nounwind "aarch64_inout_za" {13; CHECK-COMMON-LABEL: test_lazy_save_1_callee:14; CHECK-COMMON: // %bb.0:15; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill16; CHECK-COMMON-NEXT: str x19, [sp, #16] // 8-byte Spill17; CHECK-COMMON-NEXT: mov x29, sp18; CHECK-COMMON-NEXT: sub sp, sp, #1619; CHECK-COMMON-NEXT: rdsvl x8, #120; CHECK-COMMON-NEXT: mov x9, sp21; CHECK-COMMON-NEXT: msub x9, x8, x8, x922; CHECK-COMMON-NEXT: mov sp, x923; CHECK-COMMON-NEXT: sub x10, x29, #1624; CHECK-COMMON-NEXT: stp x9, x8, [x29, #-16]25; CHECK-COMMON-NEXT: msr TPIDR2_EL0, x1026; CHECK-COMMON-NEXT: bl private_za_callee27; CHECK-COMMON-NEXT: smstart za28; CHECK-COMMON-NEXT: mrs x8, TPIDR2_EL029; CHECK-COMMON-NEXT: sub x0, x29, #1630; CHECK-COMMON-NEXT: cbnz x8, .LBB0_231; CHECK-COMMON-NEXT: // %bb.1:32; CHECK-COMMON-NEXT: bl __arm_tpidr2_restore33; CHECK-COMMON-NEXT: .LBB0_2:34; CHECK-COMMON-NEXT: msr TPIDR2_EL0, xzr35; CHECK-COMMON-NEXT: mov sp, x2936; CHECK-COMMON-NEXT: ldr x19, [sp, #16] // 8-byte Reload37; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload38; CHECK-COMMON-NEXT: ret39 call void @private_za_callee()40 ret void41}42 43; Test lazy-save mechanism for multiple callees.44define void @test_lazy_save_2_callees() nounwind "aarch64_inout_za" {45; CHECK-LABEL: test_lazy_save_2_callees:46; CHECK: // %bb.0:47; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill48; CHECK-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill49; CHECK-NEXT: mov x29, sp50; CHECK-NEXT: sub sp, sp, #1651; CHECK-NEXT: rdsvl x8, #152; CHECK-NEXT: mov x9, sp53; CHECK-NEXT: msub x9, x8, x8, x954; CHECK-NEXT: mov sp, x955; CHECK-NEXT: sub x20, x29, #1656; CHECK-NEXT: stp x9, x8, [x29, #-16]57; CHECK-NEXT: msr TPIDR2_EL0, x2058; CHECK-NEXT: bl private_za_callee59; CHECK-NEXT: smstart za60; CHECK-NEXT: mrs x8, TPIDR2_EL061; CHECK-NEXT: sub x0, x29, #1662; CHECK-NEXT: cbnz x8, .LBB1_263; CHECK-NEXT: // %bb.1:64; CHECK-NEXT: bl __arm_tpidr2_restore65; CHECK-NEXT: .LBB1_2:66; CHECK-NEXT: msr TPIDR2_EL0, xzr67; CHECK-NEXT: msr TPIDR2_EL0, x2068; CHECK-NEXT: bl private_za_callee69; CHECK-NEXT: smstart za70; CHECK-NEXT: mrs x8, TPIDR2_EL071; CHECK-NEXT: sub x0, x29, #1672; CHECK-NEXT: cbnz x8, .LBB1_473; CHECK-NEXT: // %bb.3:74; CHECK-NEXT: bl __arm_tpidr2_restore75; CHECK-NEXT: .LBB1_4:76; CHECK-NEXT: msr TPIDR2_EL0, xzr77; CHECK-NEXT: mov sp, x2978; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload79; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload80; CHECK-NEXT: ret81;82; CHECK-NEWLOWERING-LABEL: test_lazy_save_2_callees:83; CHECK-NEWLOWERING: // %bb.0:84; CHECK-NEWLOWERING-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill85; CHECK-NEWLOWERING-NEXT: str x19, [sp, #16] // 8-byte Spill86; CHECK-NEWLOWERING-NEXT: mov x29, sp87; CHECK-NEWLOWERING-NEXT: sub sp, sp, #1688; CHECK-NEWLOWERING-NEXT: rdsvl x8, #189; CHECK-NEWLOWERING-NEXT: mov x9, sp90; CHECK-NEWLOWERING-NEXT: msub x9, x8, x8, x991; CHECK-NEWLOWERING-NEXT: mov sp, x992; CHECK-NEWLOWERING-NEXT: sub x10, x29, #1693; CHECK-NEWLOWERING-NEXT: stp x9, x8, [x29, #-16]94; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x1095; CHECK-NEWLOWERING-NEXT: bl private_za_callee96; CHECK-NEWLOWERING-NEXT: bl private_za_callee97; CHECK-NEWLOWERING-NEXT: smstart za98; CHECK-NEWLOWERING-NEXT: mrs x8, TPIDR2_EL099; CHECK-NEWLOWERING-NEXT: sub x0, x29, #16100; CHECK-NEWLOWERING-NEXT: cbnz x8, .LBB1_2101; CHECK-NEWLOWERING-NEXT: // %bb.1:102; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_restore103; CHECK-NEWLOWERING-NEXT: .LBB1_2:104; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr105; CHECK-NEWLOWERING-NEXT: mov sp, x29106; CHECK-NEWLOWERING-NEXT: ldr x19, [sp, #16] // 8-byte Reload107; CHECK-NEWLOWERING-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload108; CHECK-NEWLOWERING-NEXT: ret109 call void @private_za_callee()110 call void @private_za_callee()111 ret void112}113 114; Test a call of an intrinsic that gets expanded to a library call.115define float @test_lazy_save_expanded_intrinsic(float %a) nounwind "aarch64_inout_za" {116; CHECK-COMMON-LABEL: test_lazy_save_expanded_intrinsic:117; CHECK-COMMON: // %bb.0:118; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill119; CHECK-COMMON-NEXT: str x19, [sp, #16] // 8-byte Spill120; CHECK-COMMON-NEXT: mov x29, sp121; CHECK-COMMON-NEXT: sub sp, sp, #16122; CHECK-COMMON-NEXT: rdsvl x8, #1123; CHECK-COMMON-NEXT: mov x9, sp124; CHECK-COMMON-NEXT: msub x9, x8, x8, x9125; CHECK-COMMON-NEXT: mov sp, x9126; CHECK-COMMON-NEXT: sub x10, x29, #16127; CHECK-COMMON-NEXT: stp x9, x8, [x29, #-16]128; CHECK-COMMON-NEXT: msr TPIDR2_EL0, x10129; CHECK-COMMON-NEXT: bl cosf130; CHECK-COMMON-NEXT: smstart za131; CHECK-COMMON-NEXT: mrs x8, TPIDR2_EL0132; CHECK-COMMON-NEXT: sub x0, x29, #16133; CHECK-COMMON-NEXT: cbnz x8, .LBB2_2134; CHECK-COMMON-NEXT: // %bb.1:135; CHECK-COMMON-NEXT: bl __arm_tpidr2_restore136; CHECK-COMMON-NEXT: .LBB2_2:137; CHECK-COMMON-NEXT: msr TPIDR2_EL0, xzr138; CHECK-COMMON-NEXT: mov sp, x29139; CHECK-COMMON-NEXT: ldr x19, [sp, #16] // 8-byte Reload140; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload141; CHECK-COMMON-NEXT: ret142 %res = call float @llvm.cos.f32(float %a)143 ret float %res144}145 146; Test a combination of streaming-compatible -> normal call with lazy-save.147define void @test_lazy_save_and_conditional_smstart() nounwind "aarch64_inout_za" "aarch64_pstate_sm_compatible" {148; CHECK-LABEL: test_lazy_save_and_conditional_smstart:149; CHECK: // %bb.0:150; CHECK-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill151; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill152; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill153; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill154; CHECK-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill155; CHECK-NEXT: add x29, sp, #64156; CHECK-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill157; CHECK-NEXT: sub sp, sp, #16158; CHECK-NEXT: rdsvl x8, #1159; CHECK-NEXT: mov x9, sp160; CHECK-NEXT: mrs x20, SVCR161; CHECK-NEXT: msub x9, x8, x8, x9162; CHECK-NEXT: mov sp, x9163; CHECK-NEXT: sub x10, x29, #80164; CHECK-NEXT: stp x9, x8, [x29, #-80]165; CHECK-NEXT: msr TPIDR2_EL0, x10166; CHECK-NEXT: tbz w20, #0, .LBB3_2167; CHECK-NEXT: // %bb.1:168; CHECK-NEXT: smstop sm169; CHECK-NEXT: .LBB3_2:170; CHECK-NEXT: bl private_za_callee171; CHECK-NEXT: tbz w20, #0, .LBB3_4172; CHECK-NEXT: // %bb.3:173; CHECK-NEXT: smstart sm174; CHECK-NEXT: .LBB3_4:175; CHECK-NEXT: smstart za176; CHECK-NEXT: mrs x8, TPIDR2_EL0177; CHECK-NEXT: sub x0, x29, #80178; CHECK-NEXT: cbnz x8, .LBB3_6179; CHECK-NEXT: // %bb.5:180; CHECK-NEXT: bl __arm_tpidr2_restore181; CHECK-NEXT: .LBB3_6:182; CHECK-NEXT: msr TPIDR2_EL0, xzr183; CHECK-NEXT: sub sp, x29, #64184; CHECK-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload185; CHECK-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload186; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload187; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload188; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload189; CHECK-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload190; CHECK-NEXT: ret191;192; CHECK-NEWLOWERING-LABEL: test_lazy_save_and_conditional_smstart:193; CHECK-NEWLOWERING: // %bb.0:194; CHECK-NEWLOWERING-NEXT: stp d15, d14, [sp, #-96]! // 16-byte Folded Spill195; CHECK-NEWLOWERING-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill196; CHECK-NEWLOWERING-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill197; CHECK-NEWLOWERING-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill198; CHECK-NEWLOWERING-NEXT: stp x29, x30, [sp, #64] // 16-byte Folded Spill199; CHECK-NEWLOWERING-NEXT: add x29, sp, #64200; CHECK-NEWLOWERING-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill201; CHECK-NEWLOWERING-NEXT: sub sp, sp, #16202; CHECK-NEWLOWERING-NEXT: rdsvl x8, #1203; CHECK-NEWLOWERING-NEXT: mov x9, sp204; CHECK-NEWLOWERING-NEXT: msub x9, x8, x8, x9205; CHECK-NEWLOWERING-NEXT: mov sp, x9206; CHECK-NEWLOWERING-NEXT: stp x9, x8, [x29, #-80]207; CHECK-NEWLOWERING-NEXT: mrs x20, SVCR208; CHECK-NEWLOWERING-NEXT: sub x8, x29, #80209; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x8210; CHECK-NEWLOWERING-NEXT: tbz w20, #0, .LBB3_2211; CHECK-NEWLOWERING-NEXT: // %bb.1:212; CHECK-NEWLOWERING-NEXT: smstop sm213; CHECK-NEWLOWERING-NEXT: .LBB3_2:214; CHECK-NEWLOWERING-NEXT: bl private_za_callee215; CHECK-NEWLOWERING-NEXT: tbz w20, #0, .LBB3_4216; CHECK-NEWLOWERING-NEXT: // %bb.3:217; CHECK-NEWLOWERING-NEXT: smstart sm218; CHECK-NEWLOWERING-NEXT: .LBB3_4:219; CHECK-NEWLOWERING-NEXT: smstart za220; CHECK-NEWLOWERING-NEXT: mrs x8, TPIDR2_EL0221; CHECK-NEWLOWERING-NEXT: sub x0, x29, #80222; CHECK-NEWLOWERING-NEXT: cbnz x8, .LBB3_6223; CHECK-NEWLOWERING-NEXT: // %bb.5:224; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_restore225; CHECK-NEWLOWERING-NEXT: .LBB3_6:226; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr227; CHECK-NEWLOWERING-NEXT: sub sp, x29, #64228; CHECK-NEWLOWERING-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload229; CHECK-NEWLOWERING-NEXT: ldp x29, x30, [sp, #64] // 16-byte Folded Reload230; CHECK-NEWLOWERING-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload231; CHECK-NEWLOWERING-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload232; CHECK-NEWLOWERING-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload233; CHECK-NEWLOWERING-NEXT: ldp d15, d14, [sp], #96 // 16-byte Folded Reload234; CHECK-NEWLOWERING-NEXT: ret235 call void @private_za_callee()236 ret void237}238 239; Note: For the final @private_za_callee() we setup a lazy save then don't240; restore from it (since ZA is off on return). We could improve this case241; by turning ZA off before the final private ZA call.242define void @test_lazy_save_mixed_shared_and_private_callees() "aarch64_new_za"243; CHECK-LABEL: test_lazy_save_mixed_shared_and_private_callees:244; CHECK: // %bb.0: // %prelude245; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill246; CHECK-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill247; CHECK-NEXT: mov x29, sp248; CHECK-NEXT: sub sp, sp, #16249; CHECK-NEXT: .cfi_def_cfa w29, 32250; CHECK-NEXT: .cfi_offset w19, -8251; CHECK-NEXT: .cfi_offset w20, -16252; CHECK-NEXT: .cfi_offset w30, -24253; CHECK-NEXT: .cfi_offset w29, -32254; CHECK-NEXT: rdsvl x8, #1255; CHECK-NEXT: mov x9, sp256; CHECK-NEXT: msub x9, x8, x8, x9257; CHECK-NEXT: mov sp, x9258; CHECK-NEXT: stp x9, x8, [x29, #-16]259; CHECK-NEXT: mrs x8, TPIDR2_EL0260; CHECK-NEXT: cbz x8, .LBB4_2261; CHECK-NEXT: // %bb.1: // %save.za262; CHECK-NEXT: bl __arm_tpidr2_save263; CHECK-NEXT: msr TPIDR2_EL0, xzr264; CHECK-NEXT: .LBB4_2:265; CHECK-NEXT: smstart za266; CHECK-NEXT: sub x20, x29, #16267; CHECK-NEXT: zero {za}268; CHECK-NEXT: msr TPIDR2_EL0, x20269; CHECK-NEXT: bl private_za_callee270; CHECK-NEXT: smstart za271; CHECK-NEXT: mrs x8, TPIDR2_EL0272; CHECK-NEXT: sub x0, x29, #16273; CHECK-NEXT: cbnz x8, .LBB4_4274; CHECK-NEXT: // %bb.3:275; CHECK-NEXT: bl __arm_tpidr2_restore276; CHECK-NEXT: .LBB4_4:277; CHECK-NEXT: msr TPIDR2_EL0, xzr278; CHECK-NEXT: bl shared_za_callee279; CHECK-NEXT: bl preserves_za_callee280; CHECK-NEXT: msr TPIDR2_EL0, x20281; CHECK-NEXT: bl private_za_callee282; CHECK-NEXT: smstart za283; CHECK-NEXT: mrs x8, TPIDR2_EL0284; CHECK-NEXT: sub x0, x29, #16285; CHECK-NEXT: cbnz x8, .LBB4_6286; CHECK-NEXT: // %bb.5:287; CHECK-NEXT: bl __arm_tpidr2_restore288; CHECK-NEXT: .LBB4_6:289; CHECK-NEXT: msr TPIDR2_EL0, xzr290; CHECK-NEXT: smstop za291; CHECK-NEXT: mov sp, x29292; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload293; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload294; CHECK-NEXT: ret295;296; CHECK-NEWLOWERING-LABEL: test_lazy_save_mixed_shared_and_private_callees:297; CHECK-NEWLOWERING: // %bb.0:298; CHECK-NEWLOWERING-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill299; CHECK-NEWLOWERING-NEXT: str x19, [sp, #16] // 8-byte Spill300; CHECK-NEWLOWERING-NEXT: mov x29, sp301; CHECK-NEWLOWERING-NEXT: sub sp, sp, #16302; CHECK-NEWLOWERING-NEXT: .cfi_def_cfa w29, 32303; CHECK-NEWLOWERING-NEXT: .cfi_offset w19, -16304; CHECK-NEWLOWERING-NEXT: .cfi_offset w30, -24305; CHECK-NEWLOWERING-NEXT: .cfi_offset w29, -32306; CHECK-NEWLOWERING-NEXT: rdsvl x8, #1307; CHECK-NEWLOWERING-NEXT: mov x9, sp308; CHECK-NEWLOWERING-NEXT: msub x9, x8, x8, x9309; CHECK-NEWLOWERING-NEXT: mov sp, x9310; CHECK-NEWLOWERING-NEXT: stp x9, x8, [x29, #-16]311; CHECK-NEWLOWERING-NEXT: mrs x8, TPIDR2_EL0312; CHECK-NEWLOWERING-NEXT: cbz x8, .LBB4_2313; CHECK-NEWLOWERING-NEXT: // %bb.1:314; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_save315; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr316; CHECK-NEWLOWERING-NEXT: zero {za}317; CHECK-NEWLOWERING-NEXT: .LBB4_2:318; CHECK-NEWLOWERING-NEXT: smstart za319; CHECK-NEWLOWERING-NEXT: sub x8, x29, #16320; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x8321; CHECK-NEWLOWERING-NEXT: bl private_za_callee322; CHECK-NEWLOWERING-NEXT: smstart za323; CHECK-NEWLOWERING-NEXT: mrs x8, TPIDR2_EL0324; CHECK-NEWLOWERING-NEXT: sub x0, x29, #16325; CHECK-NEWLOWERING-NEXT: cbnz x8, .LBB4_4326; CHECK-NEWLOWERING-NEXT: // %bb.3:327; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_restore328; CHECK-NEWLOWERING-NEXT: .LBB4_4:329; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr330; CHECK-NEWLOWERING-NEXT: bl shared_za_callee331; CHECK-NEWLOWERING-NEXT: bl preserves_za_callee332; CHECK-NEWLOWERING-NEXT: sub x8, x29, #16333; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x8334; CHECK-NEWLOWERING-NEXT: bl private_za_callee335; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr336; CHECK-NEWLOWERING-NEXT: smstop za337; CHECK-NEWLOWERING-NEXT: mov sp, x29338; CHECK-NEWLOWERING-NEXT: ldr x19, [sp, #16] // 8-byte Reload339; CHECK-NEWLOWERING-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload340; CHECK-NEWLOWERING-NEXT: ret341{342 call void @private_za_callee()343 call void @shared_za_callee()344 call void @preserves_za_callee()345 call void @private_za_callee()346 ret void347}348 349define void @test_many_back2back_private_za_calls() "aarch64_inout_za" {350; CHECK-LABEL: test_many_back2back_private_za_calls:351; CHECK: // %bb.0:352; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill353; CHECK-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill354; CHECK-NEXT: mov x29, sp355; CHECK-NEXT: sub sp, sp, #16356; CHECK-NEXT: .cfi_def_cfa w29, 32357; CHECK-NEXT: .cfi_offset w19, -8358; CHECK-NEXT: .cfi_offset w20, -16359; CHECK-NEXT: .cfi_offset w30, -24360; CHECK-NEXT: .cfi_offset w29, -32361; CHECK-NEXT: rdsvl x8, #1362; CHECK-NEXT: mov x9, sp363; CHECK-NEXT: msub x9, x8, x8, x9364; CHECK-NEXT: mov sp, x9365; CHECK-NEXT: stp x9, x8, [x29, #-16]366; CHECK-NEXT: bl shared_za_callee367; CHECK-NEXT: sub x20, x29, #16368; CHECK-NEXT: msr TPIDR2_EL0, x20369; CHECK-NEXT: bl private_za_callee370; CHECK-NEXT: smstart za371; CHECK-NEXT: mrs x8, TPIDR2_EL0372; CHECK-NEXT: sub x0, x29, #16373; CHECK-NEXT: cbnz x8, .LBB5_2374; CHECK-NEXT: // %bb.1:375; CHECK-NEXT: bl __arm_tpidr2_restore376; CHECK-NEXT: .LBB5_2:377; CHECK-NEXT: msr TPIDR2_EL0, xzr378; CHECK-NEXT: msr TPIDR2_EL0, x20379; CHECK-NEXT: bl private_za_callee380; CHECK-NEXT: smstart za381; CHECK-NEXT: mrs x8, TPIDR2_EL0382; CHECK-NEXT: sub x0, x29, #16383; CHECK-NEXT: cbnz x8, .LBB5_4384; CHECK-NEXT: // %bb.3:385; CHECK-NEXT: bl __arm_tpidr2_restore386; CHECK-NEXT: .LBB5_4:387; CHECK-NEXT: msr TPIDR2_EL0, xzr388; CHECK-NEXT: msr TPIDR2_EL0, x20389; CHECK-NEXT: bl private_za_callee390; CHECK-NEXT: smstart za391; CHECK-NEXT: mrs x8, TPIDR2_EL0392; CHECK-NEXT: sub x0, x29, #16393; CHECK-NEXT: cbnz x8, .LBB5_6394; CHECK-NEXT: // %bb.5:395; CHECK-NEXT: bl __arm_tpidr2_restore396; CHECK-NEXT: .LBB5_6:397; CHECK-NEXT: msr TPIDR2_EL0, xzr398; CHECK-NEXT: msr TPIDR2_EL0, x20399; CHECK-NEXT: bl private_za_callee400; CHECK-NEXT: smstart za401; CHECK-NEXT: mrs x8, TPIDR2_EL0402; CHECK-NEXT: sub x0, x29, #16403; CHECK-NEXT: cbnz x8, .LBB5_8404; CHECK-NEXT: // %bb.7:405; CHECK-NEXT: bl __arm_tpidr2_restore406; CHECK-NEXT: .LBB5_8:407; CHECK-NEXT: msr TPIDR2_EL0, xzr408; CHECK-NEXT: msr TPIDR2_EL0, x20409; CHECK-NEXT: bl private_za_callee410; CHECK-NEXT: smstart za411; CHECK-NEXT: mrs x8, TPIDR2_EL0412; CHECK-NEXT: sub x0, x29, #16413; CHECK-NEXT: cbnz x8, .LBB5_10414; CHECK-NEXT: // %bb.9:415; CHECK-NEXT: bl __arm_tpidr2_restore416; CHECK-NEXT: .LBB5_10:417; CHECK-NEXT: msr TPIDR2_EL0, xzr418; CHECK-NEXT: msr TPIDR2_EL0, x20419; CHECK-NEXT: bl private_za_callee420; CHECK-NEXT: smstart za421; CHECK-NEXT: mrs x8, TPIDR2_EL0422; CHECK-NEXT: sub x0, x29, #16423; CHECK-NEXT: cbnz x8, .LBB5_12424; CHECK-NEXT: // %bb.11:425; CHECK-NEXT: bl __arm_tpidr2_restore426; CHECK-NEXT: .LBB5_12:427; CHECK-NEXT: msr TPIDR2_EL0, xzr428; CHECK-NEXT: bl shared_za_callee429; CHECK-NEXT: mov sp, x29430; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload431; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload432; CHECK-NEXT: ret433;434; CHECK-NEWLOWERING-LABEL: test_many_back2back_private_za_calls:435; CHECK-NEWLOWERING: // %bb.0:436; CHECK-NEWLOWERING-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill437; CHECK-NEWLOWERING-NEXT: str x19, [sp, #16] // 8-byte Spill438; CHECK-NEWLOWERING-NEXT: mov x29, sp439; CHECK-NEWLOWERING-NEXT: sub sp, sp, #16440; CHECK-NEWLOWERING-NEXT: .cfi_def_cfa w29, 32441; CHECK-NEWLOWERING-NEXT: .cfi_offset w19, -16442; CHECK-NEWLOWERING-NEXT: .cfi_offset w30, -24443; CHECK-NEWLOWERING-NEXT: .cfi_offset w29, -32444; CHECK-NEWLOWERING-NEXT: rdsvl x8, #1445; CHECK-NEWLOWERING-NEXT: mov x9, sp446; CHECK-NEWLOWERING-NEXT: msub x9, x8, x8, x9447; CHECK-NEWLOWERING-NEXT: mov sp, x9448; CHECK-NEWLOWERING-NEXT: stp x9, x8, [x29, #-16]449; CHECK-NEWLOWERING-NEXT: bl shared_za_callee450; CHECK-NEWLOWERING-NEXT: sub x8, x29, #16451; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x8452; CHECK-NEWLOWERING-NEXT: bl private_za_callee453; CHECK-NEWLOWERING-NEXT: bl private_za_callee454; CHECK-NEWLOWERING-NEXT: bl private_za_callee455; CHECK-NEWLOWERING-NEXT: bl private_za_callee456; CHECK-NEWLOWERING-NEXT: bl private_za_callee457; CHECK-NEWLOWERING-NEXT: bl private_za_callee458; CHECK-NEWLOWERING-NEXT: smstart za459; CHECK-NEWLOWERING-NEXT: mrs x8, TPIDR2_EL0460; CHECK-NEWLOWERING-NEXT: sub x0, x29, #16461; CHECK-NEWLOWERING-NEXT: cbnz x8, .LBB5_2462; CHECK-NEWLOWERING-NEXT: // %bb.1:463; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_restore464; CHECK-NEWLOWERING-NEXT: .LBB5_2:465; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr466; CHECK-NEWLOWERING-NEXT: bl shared_za_callee467; CHECK-NEWLOWERING-NEXT: mov sp, x29468; CHECK-NEWLOWERING-NEXT: ldr x19, [sp, #16] // 8-byte Reload469; CHECK-NEWLOWERING-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload470; CHECK-NEWLOWERING-NEXT: ret471 call void @shared_za_callee()472 call void @private_za_callee()473 call void @private_za_callee()474 call void @private_za_callee()475 call void @private_za_callee()476 call void @private_za_callee()477 call void @private_za_callee()478 call void @shared_za_callee()479 ret void480}481 482define void @test_shared_private_shared() nounwind "aarch64_inout_za" {483; CHECK-COMMON-LABEL: test_shared_private_shared:484; CHECK-COMMON: // %bb.0:485; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill486; CHECK-COMMON-NEXT: str x19, [sp, #16] // 8-byte Spill487; CHECK-COMMON-NEXT: mov x29, sp488; CHECK-COMMON-NEXT: sub sp, sp, #16489; CHECK-COMMON-NEXT: rdsvl x8, #1490; CHECK-COMMON-NEXT: mov x9, sp491; CHECK-COMMON-NEXT: msub x9, x8, x8, x9492; CHECK-COMMON-NEXT: mov sp, x9493; CHECK-COMMON-NEXT: stp x9, x8, [x29, #-16]494; CHECK-COMMON-NEXT: bl shared_za_callee495; CHECK-COMMON-NEXT: sub x8, x29, #16496; CHECK-COMMON-NEXT: msr TPIDR2_EL0, x8497; CHECK-COMMON-NEXT: bl private_za_callee498; CHECK-COMMON-NEXT: smstart za499; CHECK-COMMON-NEXT: mrs x8, TPIDR2_EL0500; CHECK-COMMON-NEXT: sub x0, x29, #16501; CHECK-COMMON-NEXT: cbnz x8, .LBB6_2502; CHECK-COMMON-NEXT: // %bb.1:503; CHECK-COMMON-NEXT: bl __arm_tpidr2_restore504; CHECK-COMMON-NEXT: .LBB6_2:505; CHECK-COMMON-NEXT: msr TPIDR2_EL0, xzr506; CHECK-COMMON-NEXT: bl shared_za_callee507; CHECK-COMMON-NEXT: mov sp, x29508; CHECK-COMMON-NEXT: ldr x19, [sp, #16] // 8-byte Reload509; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload510; CHECK-COMMON-NEXT: ret511 call void @shared_za_callee()512 call void @private_za_callee()513 call void @shared_za_callee()514 ret void515}516 517define void @test_only_shared_za() nounwind "aarch64_inout_za" {518; CHECK-COMMON-LABEL: test_only_shared_za:519; CHECK-COMMON: // %bb.0:520; CHECK-COMMON-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill521; CHECK-COMMON-NEXT: bl shared_za_callee522; CHECK-COMMON-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload523; CHECK-COMMON-NEXT: ret524 call void @shared_za_callee()525 ret void526}527 528declare i64 @shared_za_callee_i64(i64) "aarch64_inout_za"529declare i64 @private_za_callee_i64(i64)530 531define i64 @test_shared_private_shared_i64(i64 %x) nounwind "aarch64_inout_za" {532; CHECK-COMMON-LABEL: test_shared_private_shared_i64:533; CHECK-COMMON: // %bb.0:534; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill535; CHECK-COMMON-NEXT: str x19, [sp, #16] // 8-byte Spill536; CHECK-COMMON-NEXT: mov x29, sp537; CHECK-COMMON-NEXT: sub sp, sp, #16538; CHECK-COMMON-NEXT: rdsvl x8, #1539; CHECK-COMMON-NEXT: mov x9, sp540; CHECK-COMMON-NEXT: msub x9, x8, x8, x9541; CHECK-COMMON-NEXT: mov sp, x9542; CHECK-COMMON-NEXT: stp x9, x8, [x29, #-16]543; CHECK-COMMON-NEXT: bl shared_za_callee_i64544; CHECK-COMMON-NEXT: sub x8, x29, #16545; CHECK-COMMON-NEXT: msr TPIDR2_EL0, x8546; CHECK-COMMON-NEXT: bl private_za_callee_i64547; CHECK-COMMON-NEXT: mov x1, x0548; CHECK-COMMON-NEXT: smstart za549; CHECK-COMMON-NEXT: mrs x8, TPIDR2_EL0550; CHECK-COMMON-NEXT: sub x0, x29, #16551; CHECK-COMMON-NEXT: cbnz x8, .LBB8_2552; CHECK-COMMON-NEXT: // %bb.1:553; CHECK-COMMON-NEXT: bl __arm_tpidr2_restore554; CHECK-COMMON-NEXT: .LBB8_2:555; CHECK-COMMON-NEXT: mov x0, x1556; CHECK-COMMON-NEXT: msr TPIDR2_EL0, xzr557; CHECK-COMMON-NEXT: bl shared_za_callee_i64558; CHECK-COMMON-NEXT: mov sp, x29559; CHECK-COMMON-NEXT: ldr x19, [sp, #16] // 8-byte Reload560; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload561; CHECK-COMMON-NEXT: ret562 %a = call i64 @shared_za_callee_i64(i64 %x)563 %b = call i64 @private_za_callee_i64(i64 %a)564 %c = call i64 @shared_za_callee_i64(i64 %b)565 ret i64 %c566}567 568declare i64 @many_args_private_za_callee(569 i64, i64, i64, i64, i64, i64, i64, i64, i64, i64)570 571; In this example some arguments are passed on the stack, which decrements the572; stack pointer before the call -- in this test the lazy save should be setup573; before the stack decrement.574define i64 @test_many_callee_arguments(575; CHECK-LABEL: test_many_callee_arguments:576; CHECK: // %bb.0:577; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill578; CHECK-NEXT: str x19, [sp, #16] // 8-byte Spill579; CHECK-NEXT: mov x29, sp580; CHECK-NEXT: sub sp, sp, #16581; CHECK-NEXT: mov x8, sp582; CHECK-NEXT: rdsvl x9, #1583; CHECK-NEXT: msub x8, x9, x9, x8584; CHECK-NEXT: mov sp, x8585; CHECK-NEXT: ldp x10, x11, [x29, #32]586; CHECK-NEXT: sub x12, x29, #16587; CHECK-NEXT: stp x8, x9, [x29, #-16]588; CHECK-NEXT: msr TPIDR2_EL0, x12589; CHECK-NEXT: stp x10, x11, [sp, #-16]!590; CHECK-NEXT: bl many_args_private_za_callee591; CHECK-NEXT: add sp, sp, #16592; CHECK-NEXT: mov x1, x0593; CHECK-NEXT: smstart za594; CHECK-NEXT: mrs x8, TPIDR2_EL0595; CHECK-NEXT: sub x0, x29, #16596; CHECK-NEXT: cbnz x8, .LBB9_2597; CHECK-NEXT: // %bb.1:598; CHECK-NEXT: bl __arm_tpidr2_restore599; CHECK-NEXT: .LBB9_2:600; CHECK-NEXT: mov x0, x1601; CHECK-NEXT: msr TPIDR2_EL0, xzr602; CHECK-NEXT: mov sp, x29603; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Reload604; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload605; CHECK-NEXT: ret606;607; CHECK-NEWLOWERING-LABEL: test_many_callee_arguments:608; CHECK-NEWLOWERING: // %bb.0:609; CHECK-NEWLOWERING-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill610; CHECK-NEWLOWERING-NEXT: str x19, [sp, #16] // 8-byte Spill611; CHECK-NEWLOWERING-NEXT: mov x29, sp612; CHECK-NEWLOWERING-NEXT: sub sp, sp, #16613; CHECK-NEWLOWERING-NEXT: rdsvl x8, #1614; CHECK-NEWLOWERING-NEXT: mov x9, sp615; CHECK-NEWLOWERING-NEXT: msub x9, x8, x8, x9616; CHECK-NEWLOWERING-NEXT: mov sp, x9617; CHECK-NEWLOWERING-NEXT: ldp x10, x11, [x29, #32]618; CHECK-NEWLOWERING-NEXT: sub x12, x29, #16619; CHECK-NEWLOWERING-NEXT: stp x9, x8, [x29, #-16]620; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, x12621; CHECK-NEWLOWERING-NEXT: stp x10, x11, [sp, #-16]!622; CHECK-NEWLOWERING-NEXT: bl many_args_private_za_callee623; CHECK-NEWLOWERING-NEXT: add sp, sp, #16624; CHECK-NEWLOWERING-NEXT: mov x8, x0625; CHECK-NEWLOWERING-NEXT: smstart za626; CHECK-NEWLOWERING-NEXT: mrs x9, TPIDR2_EL0627; CHECK-NEWLOWERING-NEXT: sub x0, x29, #16628; CHECK-NEWLOWERING-NEXT: cbnz x9, .LBB9_2629; CHECK-NEWLOWERING-NEXT: // %bb.1:630; CHECK-NEWLOWERING-NEXT: bl __arm_tpidr2_restore631; CHECK-NEWLOWERING-NEXT: .LBB9_2:632; CHECK-NEWLOWERING-NEXT: mov x0, x8633; CHECK-NEWLOWERING-NEXT: msr TPIDR2_EL0, xzr634; CHECK-NEWLOWERING-NEXT: mov sp, x29635; CHECK-NEWLOWERING-NEXT: ldr x19, [sp, #16] // 8-byte Reload636; CHECK-NEWLOWERING-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload637; CHECK-NEWLOWERING-NEXT: ret638 i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9639) nounwind "aarch64_inout_za" {640 %ret = call i64 @many_args_private_za_callee(641 i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9)642 ret i64 %ret643}644