368 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs | FileCheck %s3; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -global-isel -global-isel-abort=2 | FileCheck %s4 5; Dynamically-sized allocation, needs a loop which can handle any size at6; runtime. The final iteration of the loop will temporarily put SP below the7; target address, but this doesn't break any of the ABI constraints on the8; stack, and also doesn't probe below the target SP value.9define void @dynamic(i64 %size, ptr %out) #0 {10; CHECK-LABEL: dynamic:11; CHECK: // %bb.0:12; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill13; CHECK-NEXT: .cfi_def_cfa_offset 1614; CHECK-NEXT: mov x29, sp15; CHECK-NEXT: .cfi_def_cfa w29, 1616; CHECK-NEXT: .cfi_offset w30, -817; CHECK-NEXT: .cfi_offset w29, -1618; CHECK-NEXT: add x9, x0, #1519; CHECK-NEXT: mov x8, sp20; CHECK-NEXT: and x9, x9, #0xfffffffffffffff021; CHECK-NEXT: sub x8, x8, x922; CHECK-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=123; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =409624; CHECK-NEXT: cmp sp, x825; CHECK-NEXT: b.le .LBB0_326; CHECK-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=127; CHECK-NEXT: str xzr, [sp]28; CHECK-NEXT: b .LBB0_129; CHECK-NEXT: .LBB0_3:30; CHECK-NEXT: mov sp, x831; CHECK-NEXT: ldr xzr, [sp]32; CHECK-NEXT: str x8, [x1]33; CHECK-NEXT: mov sp, x2934; CHECK-NEXT: .cfi_def_cfa wsp, 1635; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload36; CHECK-NEXT: .cfi_def_cfa_offset 037; CHECK-NEXT: .cfi_restore w3038; CHECK-NEXT: .cfi_restore w2939; CHECK-NEXT: ret40 %v = alloca i8, i64 %size, align 141 store ptr %v, ptr %out, align 842 ret void43}44 45; This function has a fixed-size stack slot and a dynamic one. The fixed size46; slot isn't large enough that we would normally probe it, but we need to do so47; here otherwise the gap between the CSR save and the first probe of the48; dynamic allocation could be too far apart when the size of the dynamic49; allocation is close to the guard size.50define void @dynamic_fixed(i64 %size, ptr %out1, ptr %out2) #0 {51; CHECK-LABEL: dynamic_fixed:52; CHECK: // %bb.0:53; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill54; CHECK-NEXT: .cfi_def_cfa_offset 1655; CHECK-NEXT: mov x29, sp56; CHECK-NEXT: .cfi_def_cfa w29, 1657; CHECK-NEXT: .cfi_offset w30, -858; CHECK-NEXT: .cfi_offset w29, -1659; CHECK-NEXT: str xzr, [sp, #-64]!60; CHECK-NEXT: add x9, x0, #1561; CHECK-NEXT: mov x8, sp62; CHECK-DAG: sub x10, x29, #6463; CHECK-DAG: and x9, x9, #0xfffffffffffffff064; CHECK-NOT: INVALID_TO_BREAK_UP_CHECK_DAG65; CHECK-DAG: str x10, [x1]66; CHECK-DAG: sub x8, x8, x967; CHECK-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=168; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =409669; CHECK-NEXT: cmp sp, x870; CHECK-NEXT: b.le .LBB1_371; CHECK-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=172; CHECK-NEXT: str xzr, [sp]73; CHECK-NEXT: b .LBB1_174; CHECK-NEXT: .LBB1_3:75; CHECK-NEXT: mov sp, x876; CHECK-NEXT: ldr xzr, [sp]77; CHECK-NEXT: str x8, [x2]78; CHECK-NEXT: mov sp, x2979; CHECK-NEXT: .cfi_def_cfa wsp, 1680; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload81; CHECK-NEXT: .cfi_def_cfa_offset 082; CHECK-NEXT: .cfi_restore w3083; CHECK-NEXT: .cfi_restore w2984; CHECK-NEXT: ret85 %v1 = alloca i8, i64 64, align 186 store ptr %v1, ptr %out1, align 887 %v2 = alloca i8, i64 %size, align 188 store ptr %v2, ptr %out2, align 889 ret void90}91 92; Dynamic allocation, with an alignment requirement greater than the alignment93; of SP. Done by ANDing the target SP with a constant to align it down, then94; doing the loop as normal. Note that we also re-align the stack in the prolog,95; which isn't actually needed because the only aligned allocations are dynamic,96; this is done even without stack probing.97define void @dynamic_align_64(i64 %size, ptr %out) #0 {98; CHECK-LABEL: dynamic_align_64:99; CHECK: // %bb.0:100; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill101; CHECK-NEXT: .cfi_def_cfa_offset 32102; CHECK-NEXT: str x19, [sp, #16] // 8-byte Spill103; CHECK-NEXT: mov x29, sp104; CHECK-NEXT: .cfi_def_cfa w29, 32105; CHECK-NEXT: .cfi_offset w19, -16106; CHECK-NEXT: .cfi_offset w30, -24107; CHECK-NEXT: .cfi_offset w29, -32108; CHECK-NEXT: sub x9, sp, #32109; CHECK-NEXT: and sp, x9, #0xffffffffffffffc0110; CHECK-NEXT: add x9, x0, #15111; CHECK-NEXT: mov x8, sp112; CHECK-DAG: str xzr, [sp]113; CHECK-DAG: and x9, x9, #0xfffffffffffffff0114; CHECK-NOT: INVALID_TO_BREAK_UP_CHECK_DAG115; CHECK-DAG: mov x19, sp116; CHECK-DAG: sub x8, x8, x9117; CHECK-NEXT: and x8, x8, #0xffffffffffffffc0118; CHECK-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1119; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096120; CHECK-NEXT: cmp sp, x8121; CHECK-NEXT: b.le .LBB2_3122; CHECK-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1123; CHECK-NEXT: str xzr, [sp]124; CHECK-NEXT: b .LBB2_1125; CHECK-NEXT: .LBB2_3:126; CHECK-NEXT: mov sp, x8127; CHECK-NEXT: ldr xzr, [sp]128; CHECK-NEXT: str x8, [x1]129; CHECK-NEXT: mov sp, x29130; CHECK-NEXT: .cfi_def_cfa wsp, 32131; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Reload132; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload133; CHECK-NEXT: .cfi_def_cfa_offset 0134; CHECK-NEXT: .cfi_restore w19135; CHECK-NEXT: .cfi_restore w30136; CHECK-NEXT: .cfi_restore w29137; CHECK-NEXT: ret138 %v = alloca i8, i64 %size, align 64139 store ptr %v, ptr %out, align 8140 ret void141}142 143; Dynamic allocation, with an alignment greater than the stack guard size. The144; only difference to the dynamic allocation is the constant used for aligning145; the target SP, the loop will probe the whole allocation without needing to146; know about the alignment padding.147define void @dynamic_align_8192(i64 %size, ptr %out) #0 {148; CHECK-LABEL: dynamic_align_8192:149; CHECK: // %bb.0:150; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill151; CHECK-NEXT: .cfi_def_cfa_offset 32152; CHECK-NEXT: str x19, [sp, #16] // 8-byte Spill153; CHECK-NEXT: mov x29, sp154; CHECK-NEXT: .cfi_def_cfa w29, 32155; CHECK-NEXT: .cfi_offset w19, -16156; CHECK-NEXT: .cfi_offset w30, -24157; CHECK-NEXT: .cfi_offset w29, -32158; CHECK-NEXT: sub x9, sp, #1, lsl #12 // =4096159; CHECK-NEXT: sub x9, x9, #4064160; CHECK-NEXT: and x9, x9, #0xffffffffffffe000161; CHECK-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1162; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096163; CHECK-NEXT: cmp sp, x9164; CHECK-NEXT: b.le .LBB3_3165; CHECK-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1166; CHECK-NEXT: str xzr, [sp]167; CHECK-NEXT: b .LBB3_1168; CHECK-NEXT: .LBB3_3:169; CHECK-NEXT: mov sp, x9170; CHECK-NEXT: add x9, x0, #15171; CHECK-NEXT: mov x8, sp172; CHECK-DAG: ldr xzr, [sp]173; CHECK-DAG: and x9, x9, #0xfffffffffffffff0174; CHECK-NOT: INVALID_TO_BREAK_UP_CHECK_DAG175; CHECK-DAG: mov x19, sp176; CHECK-DAG: sub x8, x8, x9177; CHECK-NEXT: and x8, x8, #0xffffffffffffe000178; CHECK-NEXT: .LBB3_4: // =>This Inner Loop Header: Depth=1179; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096180; CHECK-NEXT: cmp sp, x8181; CHECK-NEXT: b.le .LBB3_6182; CHECK-NEXT: // %bb.5: // in Loop: Header=BB3_4 Depth=1183; CHECK-NEXT: str xzr, [sp]184; CHECK-NEXT: b .LBB3_4185; CHECK-NEXT: .LBB3_6:186; CHECK-NEXT: mov sp, x8187; CHECK-NEXT: ldr xzr, [sp]188; CHECK-NEXT: str x8, [x1]189; CHECK-NEXT: mov sp, x29190; CHECK-NEXT: .cfi_def_cfa wsp, 32191; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Reload192; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload193; CHECK-NEXT: .cfi_def_cfa_offset 0194; CHECK-NEXT: .cfi_restore w19195; CHECK-NEXT: .cfi_restore w30196; CHECK-NEXT: .cfi_restore w29197; CHECK-NEXT: ret198 %v = alloca i8, i64 %size, align 8192199 store ptr %v, ptr %out, align 8200 ret void201}202 203; For 64k guard pages, the only difference is the constant subtracted from SP204; in the loop.205define void @dynamic_64k_guard(i64 %size, ptr %out) #0 "stack-probe-size"="65536" {206; CHECK-LABEL: dynamic_64k_guard:207; CHECK: // %bb.0:208; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill209; CHECK-NEXT: .cfi_def_cfa_offset 16210; CHECK-NEXT: mov x29, sp211; CHECK-NEXT: .cfi_def_cfa w29, 16212; CHECK-NEXT: .cfi_offset w30, -8213; CHECK-NEXT: .cfi_offset w29, -16214; CHECK-NEXT: add x9, x0, #15215; CHECK-NEXT: mov x8, sp216; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0217; CHECK-NEXT: sub x8, x8, x9218; CHECK-NEXT: .LBB4_1: // =>This Inner Loop Header: Depth=1219; CHECK-NEXT: sub sp, sp, #16, lsl #12 // =65536220; CHECK-NEXT: cmp sp, x8221; CHECK-NEXT: b.le .LBB4_3222; CHECK-NEXT: // %bb.2: // in Loop: Header=BB4_1 Depth=1223; CHECK-NEXT: str xzr, [sp]224; CHECK-NEXT: b .LBB4_1225; CHECK-NEXT: .LBB4_3:226; CHECK-NEXT: mov sp, x8227; CHECK-NEXT: ldr xzr, [sp]228; CHECK-NEXT: str x8, [x1]229; CHECK-NEXT: mov sp, x29230; CHECK-NEXT: .cfi_def_cfa wsp, 16231; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload232; CHECK-NEXT: .cfi_def_cfa_offset 0233; CHECK-NEXT: .cfi_restore w30234; CHECK-NEXT: .cfi_restore w29235; CHECK-NEXT: ret236 %v = alloca i8, i64 %size, align 1237 store ptr %v, ptr %out, align 8238 ret void239}240 241; If a function has variable-sized stack objects, then any function calls which242; need to pass arguments on the stack must allocate the stack space for them243; dynamically, to ensure they are at the bottom of the frame. We need to probe244; that space when it is larger than the unprobed space allowed by the ABI (1024245; bytes), so this needs a very large number of arguments.246define void @no_reserved_call_frame(i64 %n) #0 {247; CHECK-LABEL: no_reserved_call_frame:248; CHECK: // %bb.0: // %entry249; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill250; CHECK-NEXT: .cfi_def_cfa_offset 16251; CHECK-NEXT: mov x29, sp252; CHECK-NEXT: .cfi_def_cfa w29, 16253; CHECK-NEXT: .cfi_offset w30, -8254; CHECK-NEXT: .cfi_offset w29, -16255; CHECK-NEXT: lsl x9, x0, #2256; CHECK-NEXT: mov x8, sp257; CHECK-NEXT: add x9, x9, #15258; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0259; CHECK-NEXT: sub x0, x8, x9260; CHECK-NEXT: .LBB5_1: // %entry261; CHECK-NEXT: // =>This Inner Loop Header: Depth=1262; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096263; CHECK-NEXT: cmp sp, x0264; CHECK-NEXT: b.le .LBB5_3265; CHECK-NEXT: // %bb.2: // %entry266; CHECK-NEXT: // in Loop: Header=BB5_1 Depth=1267; CHECK-NEXT: str xzr, [sp]268; CHECK-NEXT: b .LBB5_1269; CHECK-NEXT: .LBB5_3: // %entry270; CHECK-NEXT: mov sp, x0271; CHECK-NEXT: ldr xzr, [sp]272; CHECK-NEXT: sub sp, sp, #1104273; CHECK-NEXT: str xzr, [sp]274; CHECK-NEXT: bl callee_stack_args275; CHECK-NEXT: add sp, sp, #1104276; CHECK-NEXT: mov sp, x29277; CHECK-NEXT: .cfi_def_cfa wsp, 16278; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload279; CHECK-NEXT: .cfi_def_cfa_offset 0280; CHECK-NEXT: .cfi_restore w30281; CHECK-NEXT: .cfi_restore w29282; CHECK-NEXT: ret283entry:284 %v = alloca i32, i64 %n285 call void @callee_stack_args(ptr %v, [138 x i64] undef)286 ret void287}288 289; Same as above but without a variable-sized allocation, so the reserved call290; frame can be folded into the fixed-size allocation in the prologue.291define void @reserved_call_frame(i64 %n) #0 {292; CHECK-LABEL: reserved_call_frame:293; CHECK: // %bb.0: // %entry294; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill295; CHECK-NEXT: .cfi_def_cfa_offset 32296; CHECK-NEXT: str x28, [sp, #16] // 8-byte Spill297; CHECK-NEXT: mov x29, sp298; CHECK-NEXT: .cfi_def_cfa w29, 32299; CHECK-NEXT: .cfi_offset w28, -16300; CHECK-NEXT: .cfi_offset w30, -24301; CHECK-NEXT: .cfi_offset w29, -32302; CHECK-NEXT: sub sp, sp, #1504303; CHECK-NEXT: add x0, sp, #1104304; CHECK-NEXT: str xzr, [sp]305; CHECK-NEXT: bl callee_stack_args306; CHECK-NEXT: add sp, sp, #1504307; CHECK-NEXT: .cfi_def_cfa wsp, 32308; CHECK-NEXT: ldr x28, [sp, #16] // 8-byte Reload309; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload310; CHECK-NEXT: .cfi_def_cfa_offset 0311; CHECK-NEXT: .cfi_restore w28312; CHECK-NEXT: .cfi_restore w30313; CHECK-NEXT: .cfi_restore w29314; CHECK-NEXT: ret315entry:316 %v = alloca i32, i64 100317 call void @callee_stack_args(ptr %v, [138 x i64] undef)318 ret void319}320 321declare void @callee_stack_args(ptr, [138 x i64])322 323; Dynamic allocation of SVE vectors324define void @dynamic_sve(i64 %size, ptr %out) #0 "target-features"="+sve" {325; CHECK-LABEL: dynamic_sve:326; CHECK: // %bb.0:327; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill328; CHECK-NEXT: .cfi_def_cfa_offset 32329; CHECK-NEXT: str x19, [sp, #16] // 8-byte Spill330; CHECK-NEXT: mov x29, sp331; CHECK-NEXT: .cfi_def_cfa w29, 32332; CHECK-NEXT: .cfi_offset w19, -16333; CHECK-NEXT: .cfi_offset w30, -24334; CHECK-NEXT: .cfi_offset w29, -32335; CHECK-NEXT: rdvl x9, #1336; CHECK-NEXT: mov x10, #15 // =0xf337; CHECK-DAG: mov x8, sp338; CHECK-DAG: madd x9, x0, x9, x10339; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0340; CHECK-NEXT: sub x8, x8, x9341; CHECK-NEXT: .LBB7_1: // =>This Inner Loop Header: Depth=1342; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096343; CHECK-NEXT: cmp sp, x8344; CHECK-NEXT: b.le .LBB7_3345; CHECK-NEXT: // %bb.2: // in Loop: Header=BB7_1 Depth=1346; CHECK-NEXT: str xzr, [sp]347; CHECK-NEXT: b .LBB7_1348; CHECK-NEXT: .LBB7_3:349; CHECK-NEXT: mov sp, x8350; CHECK-NEXT: ldr xzr, [sp]351; CHECK-NEXT: str x8, [x1]352; CHECK-NEXT: mov sp, x29353; CHECK-NEXT: .cfi_def_cfa wsp, 32354; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Reload355; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload356; CHECK-NEXT: .cfi_def_cfa_offset 0357; CHECK-NEXT: .cfi_restore w19358; CHECK-NEXT: .cfi_restore w30359; CHECK-NEXT: .cfi_restore w29360; CHECK-NEXT: ret361 %v = alloca <vscale x 4 x float>, i64 %size, align 16362 store ptr %v, ptr %out, align 8363 ret void364}365 366attributes #0 = { uwtable(async) "probe-stack"="inline-asm" "frame-pointer"="none" }367 368