725 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs | FileCheck %s3; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -global-isel -global-isel-abort=2 | FileCheck %s4 5; Test prolog sequences for stack probing when SVE objects are involved.6 7; The space for SVE objects needs probing in the general case, because8; the stack adjustment may happen to be too big (i.e. greater than the9; probe size) to allocate with a single `addvl`.10; When we do know that the stack adjustment cannot exceed the probe size11; we can avoid emitting a probe loop and emit a simple `addvl; str`12; sequence instead.13 14define void @sve_1_vector(ptr %out) #0 {15; CHECK-LABEL: sve_1_vector:16; CHECK: // %bb.0: // %entry17; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill18; CHECK-NEXT: .cfi_def_cfa_offset 1619; CHECK-NEXT: .cfi_offset w29, -1620; CHECK-NEXT: addvl sp, sp, #-121; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG22; CHECK-NEXT: addvl sp, sp, #123; CHECK-NEXT: .cfi_def_cfa wsp, 1624; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload25; CHECK-NEXT: .cfi_def_cfa_offset 026; CHECK-NEXT: .cfi_restore w2927; CHECK-NEXT: ret28entry:29 %vec = alloca <vscale x 4 x float>, align 1630 ret void31}32 33; As above, but with 4 SVE vectors of stack space.34define void @sve_4_vector(ptr %out) #0 {35; CHECK-LABEL: sve_4_vector:36; CHECK: // %bb.0: // %entry37; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill38; CHECK-NEXT: .cfi_def_cfa_offset 1639; CHECK-NEXT: .cfi_offset w29, -1640; CHECK-NEXT: addvl sp, sp, #-441; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG42; CHECK-NEXT: addvl sp, sp, #443; CHECK-NEXT: .cfi_def_cfa wsp, 1644; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload45; CHECK-NEXT: .cfi_def_cfa_offset 046; CHECK-NEXT: .cfi_restore w2947; CHECK-NEXT: ret48entry:49 %vec1 = alloca <vscale x 4 x float>, align 1650 %vec2 = alloca <vscale x 4 x float>, align 1651 %vec3 = alloca <vscale x 4 x float>, align 1652 %vec4 = alloca <vscale x 4 x float>, align 1653 ret void54}55 56; As above, but with 16 SVE vectors of stack space.57; The stack adjustment is less than or equal to 16 x 256 = 4096, so58; we can allocate the locals at once.59define void @sve_16_vector(ptr %out) #0 {60; CHECK-LABEL: sve_16_vector:61; CHECK: // %bb.0: // %entry62; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill63; CHECK-NEXT: .cfi_def_cfa_offset 1664; CHECK-NEXT: .cfi_offset w29, -1665; CHECK-NEXT: addvl sp, sp, #-1666; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x01, 0x1e, 0x22 // sp + 16 + 128 * VG67; CHECK-NEXT: str xzr, [sp]68; CHECK-NEXT: addvl sp, sp, #1669; CHECK-NEXT: .cfi_def_cfa wsp, 1670; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload71; CHECK-NEXT: .cfi_def_cfa_offset 072; CHECK-NEXT: .cfi_restore w2973; CHECK-NEXT: ret74entry:75 %vec1 = alloca <vscale x 4 x float>, align 1676 %vec2 = alloca <vscale x 4 x float>, align 1677 %vec3 = alloca <vscale x 4 x float>, align 1678 %vec4 = alloca <vscale x 4 x float>, align 1679 %vec5 = alloca <vscale x 4 x float>, align 1680 %vec6 = alloca <vscale x 4 x float>, align 1681 %vec7 = alloca <vscale x 4 x float>, align 1682 %vec8 = alloca <vscale x 4 x float>, align 1683 %vec9 = alloca <vscale x 4 x float>, align 1684 %vec10 = alloca <vscale x 4 x float>, align 1685 %vec11 = alloca <vscale x 4 x float>, align 1686 %vec12 = alloca <vscale x 4 x float>, align 1687 %vec13 = alloca <vscale x 4 x float>, align 1688 %vec14 = alloca <vscale x 4 x float>, align 1689 %vec15 = alloca <vscale x 4 x float>, align 1690 %vec16 = alloca <vscale x 4 x float>, align 1691 ret void92}93 94; As above, but with 17 SVE vectors of stack space. Now we need95; a probing loops since stack adjustment may be greater than96; the probe size (17 x 256 = 4354 bytes)97; TODO: Allocating `k*16+r` SVE vectors can be unrolled into98; emiting the `k + r` sequences of `addvl sp, sp, #-N; str xzr, [sp]`99define void @sve_17_vector(ptr %out) #0 {100; CHECK-LABEL: sve_17_vector:101; CHECK: // %bb.0: // %entry102; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill103; CHECK-NEXT: .cfi_def_cfa_offset 16104; CHECK-NEXT: .cfi_offset w29, -16105; CHECK-NEXT: addvl x9, sp, #-17106; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x88, 0x01, 0x1e, 0x22 // $x9 + 16 + 136 * VG107; CHECK-NEXT: .LBB3_1: // %entry108; CHECK-NEXT: // =>This Inner Loop Header: Depth=1109; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096110; CHECK-NEXT: cmp sp, x9111; CHECK-NEXT: b.le .LBB3_3112; CHECK-NEXT: // %bb.2: // %entry113; CHECK-NEXT: // in Loop: Header=BB3_1 Depth=1114; CHECK-NEXT: str xzr, [sp]115; CHECK-NEXT: b .LBB3_1116; CHECK-NEXT: .LBB3_3: // %entry117; CHECK-NEXT: mov sp, x9118; CHECK-NEXT: ldr xzr, [sp]119; CHECK-NEXT: .cfi_def_cfa_register wsp120; CHECK-NEXT: addvl sp, sp, #17121; CHECK-NEXT: .cfi_def_cfa wsp, 16122; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload123; CHECK-NEXT: .cfi_def_cfa_offset 0124; CHECK-NEXT: .cfi_restore w29125; CHECK-NEXT: ret126entry:127 %vec1 = alloca <vscale x 4 x float>, align 16128 %vec2 = alloca <vscale x 4 x float>, align 16129 %vec3 = alloca <vscale x 4 x float>, align 16130 %vec4 = alloca <vscale x 4 x float>, align 16131 %vec5 = alloca <vscale x 4 x float>, align 16132 %vec6 = alloca <vscale x 4 x float>, align 16133 %vec7 = alloca <vscale x 4 x float>, align 16134 %vec8 = alloca <vscale x 4 x float>, align 16135 %vec9 = alloca <vscale x 4 x float>, align 16136 %vec10 = alloca <vscale x 4 x float>, align 16137 %vec11 = alloca <vscale x 4 x float>, align 16138 %vec12 = alloca <vscale x 4 x float>, align 16139 %vec13 = alloca <vscale x 4 x float>, align 16140 %vec14 = alloca <vscale x 4 x float>, align 16141 %vec15 = alloca <vscale x 4 x float>, align 16142 %vec16 = alloca <vscale x 4 x float>, align 16143 %vec17 = alloca <vscale x 4 x float>, align 16144 ret void145}146 147; Space for callee-saved SVE register is allocated similarly to allocating148; space for SVE locals. When we know the stack adjustment cannot exceed the149; probe size we can skip the explict probe, since saving SVE registers serves150; as an implicit probe.151define void @sve_1v_csr(<vscale x 4 x float> %a) #0 {152; CHECK-LABEL: sve_1v_csr:153; CHECK: // %bb.0: // %entry154; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill155; CHECK-NEXT: .cfi_def_cfa_offset 16156; CHECK-NEXT: .cfi_offset w29, -16157; CHECK-NEXT: addvl sp, sp, #-1158; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG159; CHECK-NEXT: str z8, [sp] // 16-byte Folded Spill160; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16161; CHECK-NEXT: //APP162; CHECK-NEXT: //NO_APP163; CHECK-NEXT: ldr z8, [sp] // 16-byte Folded Reload164; CHECK-NEXT: addvl sp, sp, #1165; CHECK-NEXT: .cfi_def_cfa wsp, 16166; CHECK-NEXT: .cfi_restore z8167; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload168; CHECK-NEXT: .cfi_def_cfa_offset 0169; CHECK-NEXT: .cfi_restore w29170; CHECK-NEXT: ret171entry:172 call void asm sideeffect "", "~{z8}" ()173 ret void174}175 176define void @sve_4v_csr(<vscale x 4 x float> %a) #0 {177; CHECK-LABEL: sve_4v_csr:178; CHECK: // %bb.0: // %entry179; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill180; CHECK-NEXT: .cfi_def_cfa_offset 16181; CHECK-NEXT: .cfi_offset w29, -16182; CHECK-NEXT: addvl sp, sp, #-4183; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG184; CHECK-NEXT: str z11, [sp] // 16-byte Folded Spill185; CHECK-NEXT: str z10, [sp, #1, mul vl] // 16-byte Folded Spill186; CHECK-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill187; CHECK-NEXT: str z8, [sp, #3, mul vl] // 16-byte Folded Spill188; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16189; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16190; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16191; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 32 * VG - 16192; CHECK-NEXT: //APP193; CHECK-NEXT: //NO_APP194; CHECK-NEXT: ldr z11, [sp] // 16-byte Folded Reload195; CHECK-NEXT: ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload196; CHECK-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload197; CHECK-NEXT: ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload198; CHECK-NEXT: addvl sp, sp, #4199; CHECK-NEXT: .cfi_def_cfa wsp, 16200; CHECK-NEXT: .cfi_restore z8201; CHECK-NEXT: .cfi_restore z9202; CHECK-NEXT: .cfi_restore z10203; CHECK-NEXT: .cfi_restore z11204; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload205; CHECK-NEXT: .cfi_def_cfa_offset 0206; CHECK-NEXT: .cfi_restore w29207; CHECK-NEXT: ret208entry:209 call void asm sideeffect "", "~{z8},~{z9},~{z10},~{z11}" ()210 ret void211}212 213define void @sve_16v_csr(<vscale x 4 x float> %a) #0 {214; CHECK-LABEL: sve_16v_csr:215; CHECK: // %bb.0: // %entry216; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill217; CHECK-NEXT: .cfi_def_cfa_offset 16218; CHECK-NEXT: .cfi_offset w29, -16219; CHECK-NEXT: addvl sp, sp, #-16220; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x01, 0x1e, 0x22 // sp + 16 + 128 * VG221; CHECK-NEXT: str xzr, [sp]222; CHECK-NEXT: str z23, [sp] // 16-byte Folded Spill223; CHECK-NEXT: str z22, [sp, #1, mul vl] // 16-byte Folded Spill224; CHECK-NEXT: str z21, [sp, #2, mul vl] // 16-byte Folded Spill225; CHECK-NEXT: str z20, [sp, #3, mul vl] // 16-byte Folded Spill226; CHECK-NEXT: str z19, [sp, #4, mul vl] // 16-byte Folded Spill227; CHECK-NEXT: str z18, [sp, #5, mul vl] // 16-byte Folded Spill228; CHECK-NEXT: str z17, [sp, #6, mul vl] // 16-byte Folded Spill229; CHECK-NEXT: str z16, [sp, #7, mul vl] // 16-byte Folded Spill230; CHECK-NEXT: str z15, [sp, #8, mul vl] // 16-byte Folded Spill231; CHECK-NEXT: str z14, [sp, #9, mul vl] // 16-byte Folded Spill232; CHECK-NEXT: str z13, [sp, #10, mul vl] // 16-byte Folded Spill233; CHECK-NEXT: str z12, [sp, #11, mul vl] // 16-byte Folded Spill234; CHECK-NEXT: str z11, [sp, #12, mul vl] // 16-byte Folded Spill235; CHECK-NEXT: str z10, [sp, #13, mul vl] // 16-byte Folded Spill236; CHECK-NEXT: str z9, [sp, #14, mul vl] // 16-byte Folded Spill237; CHECK-NEXT: str z8, [sp, #15, mul vl] // 16-byte Folded Spill238; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16239; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16240; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16241; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 32 * VG - 16242; CHECK-NEXT: .cfi_escape 0x10, 0x4c, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x40, 0x1c // $d12 @ cfa - 40 * VG - 16243; CHECK-NEXT: .cfi_escape 0x10, 0x4d, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x40, 0x1c // $d13 @ cfa - 48 * VG - 16244; CHECK-NEXT: .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16245; CHECK-NEXT: .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16246; CHECK-NEXT: //APP247; CHECK-NEXT: //NO_APP248; CHECK-NEXT: ldr z23, [sp] // 16-byte Folded Reload249; CHECK-NEXT: ldr z22, [sp, #1, mul vl] // 16-byte Folded Reload250; CHECK-NEXT: ldr z21, [sp, #2, mul vl] // 16-byte Folded Reload251; CHECK-NEXT: ldr z20, [sp, #3, mul vl] // 16-byte Folded Reload252; CHECK-NEXT: ldr z19, [sp, #4, mul vl] // 16-byte Folded Reload253; CHECK-NEXT: ldr z18, [sp, #5, mul vl] // 16-byte Folded Reload254; CHECK-NEXT: ldr z17, [sp, #6, mul vl] // 16-byte Folded Reload255; CHECK-NEXT: ldr z16, [sp, #7, mul vl] // 16-byte Folded Reload256; CHECK-NEXT: ldr z15, [sp, #8, mul vl] // 16-byte Folded Reload257; CHECK-NEXT: ldr z14, [sp, #9, mul vl] // 16-byte Folded Reload258; CHECK-NEXT: ldr z13, [sp, #10, mul vl] // 16-byte Folded Reload259; CHECK-NEXT: ldr z12, [sp, #11, mul vl] // 16-byte Folded Reload260; CHECK-NEXT: ldr z11, [sp, #12, mul vl] // 16-byte Folded Reload261; CHECK-NEXT: ldr z10, [sp, #13, mul vl] // 16-byte Folded Reload262; CHECK-NEXT: ldr z9, [sp, #14, mul vl] // 16-byte Folded Reload263; CHECK-NEXT: ldr z8, [sp, #15, mul vl] // 16-byte Folded Reload264; CHECK-NEXT: addvl sp, sp, #16265; CHECK-NEXT: .cfi_def_cfa wsp, 16266; CHECK-NEXT: .cfi_restore z8267; CHECK-NEXT: .cfi_restore z9268; CHECK-NEXT: .cfi_restore z10269; CHECK-NEXT: .cfi_restore z11270; CHECK-NEXT: .cfi_restore z12271; CHECK-NEXT: .cfi_restore z13272; CHECK-NEXT: .cfi_restore z14273; CHECK-NEXT: .cfi_restore z15274; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload275; CHECK-NEXT: .cfi_def_cfa_offset 0276; CHECK-NEXT: .cfi_restore w29277; CHECK-NEXT: ret278entry:279 call void asm sideeffect "", "~{z8},~{z9},~{z10},~{z11},~{z12},~{z13},~{z14},~{z15},~{z16},~{z17},~{z18},~{z19},~{z20},~{z21},~{z22},~{z23}" ()280 ret void281}282 283define void @sve_1p_csr(<vscale x 4 x float> %a) #0 {284; CHECK-LABEL: sve_1p_csr:285; CHECK: // %bb.0: // %entry286; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill287; CHECK-NEXT: .cfi_def_cfa_offset 16288; CHECK-NEXT: .cfi_offset w29, -16289; CHECK-NEXT: addvl sp, sp, #-1290; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG291; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill292; CHECK-NEXT: //APP293; CHECK-NEXT: //NO_APP294; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload295; CHECK-NEXT: addvl sp, sp, #1296; CHECK-NEXT: .cfi_def_cfa wsp, 16297; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload298; CHECK-NEXT: .cfi_def_cfa_offset 0299; CHECK-NEXT: .cfi_restore w29300; CHECK-NEXT: ret301entry:302 call void asm sideeffect "", "~{p8}" ()303 ret void304}305 306define void @sve_4p_csr(<vscale x 4 x float> %a) #0 {307; CHECK-LABEL: sve_4p_csr:308; CHECK: // %bb.0: // %entry309; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill310; CHECK-NEXT: .cfi_def_cfa_offset 16311; CHECK-NEXT: .cfi_offset w29, -16312; CHECK-NEXT: addvl sp, sp, #-1313; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG314; CHECK-NEXT: str p11, [sp, #4, mul vl] // 2-byte Spill315; CHECK-NEXT: str p10, [sp, #5, mul vl] // 2-byte Spill316; CHECK-NEXT: str p9, [sp, #6, mul vl] // 2-byte Spill317; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill318; CHECK-NEXT: //APP319; CHECK-NEXT: //NO_APP320; CHECK-NEXT: ldr p11, [sp, #4, mul vl] // 2-byte Reload321; CHECK-NEXT: ldr p10, [sp, #5, mul vl] // 2-byte Reload322; CHECK-NEXT: ldr p9, [sp, #6, mul vl] // 2-byte Reload323; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload324; CHECK-NEXT: addvl sp, sp, #1325; CHECK-NEXT: .cfi_def_cfa wsp, 16326; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload327; CHECK-NEXT: .cfi_def_cfa_offset 0328; CHECK-NEXT: .cfi_restore w29329; CHECK-NEXT: ret330entry:331 call void asm sideeffect "", "~{p8},~{p9},~{p10},~{p11}" ()332 ret void333}334 335define void @sve_16v_1p_csr(<vscale x 4 x float> %a) #0 {336; CHECK-LABEL: sve_16v_1p_csr:337; CHECK: // %bb.0: // %entry338; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill339; CHECK-NEXT: .cfi_def_cfa_offset 16340; CHECK-NEXT: .cfi_offset w29, -16341; CHECK-NEXT: addvl x9, sp, #-17342; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x88, 0x01, 0x1e, 0x22 // $x9 + 16 + 136 * VG343; CHECK-NEXT: .LBB9_1: // %entry344; CHECK-NEXT: // =>This Inner Loop Header: Depth=1345; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096346; CHECK-NEXT: cmp sp, x9347; CHECK-NEXT: b.le .LBB9_3348; CHECK-NEXT: // %bb.2: // %entry349; CHECK-NEXT: // in Loop: Header=BB9_1 Depth=1350; CHECK-NEXT: str xzr, [sp]351; CHECK-NEXT: b .LBB9_1352; CHECK-NEXT: .LBB9_3: // %entry353; CHECK-NEXT: mov sp, x9354; CHECK-NEXT: ldr xzr, [sp]355; CHECK-NEXT: .cfi_def_cfa_register wsp356; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill357; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill358; CHECK-NEXT: str z22, [sp, #2, mul vl] // 16-byte Folded Spill359; CHECK-NEXT: str z21, [sp, #3, mul vl] // 16-byte Folded Spill360; CHECK-NEXT: str z20, [sp, #4, mul vl] // 16-byte Folded Spill361; CHECK-NEXT: str z19, [sp, #5, mul vl] // 16-byte Folded Spill362; CHECK-NEXT: str z18, [sp, #6, mul vl] // 16-byte Folded Spill363; CHECK-NEXT: str z17, [sp, #7, mul vl] // 16-byte Folded Spill364; CHECK-NEXT: str z16, [sp, #8, mul vl] // 16-byte Folded Spill365; CHECK-NEXT: str z15, [sp, #9, mul vl] // 16-byte Folded Spill366; CHECK-NEXT: str z14, [sp, #10, mul vl] // 16-byte Folded Spill367; CHECK-NEXT: str z13, [sp, #11, mul vl] // 16-byte Folded Spill368; CHECK-NEXT: str z12, [sp, #12, mul vl] // 16-byte Folded Spill369; CHECK-NEXT: str z11, [sp, #13, mul vl] // 16-byte Folded Spill370; CHECK-NEXT: str z10, [sp, #14, mul vl] // 16-byte Folded Spill371; CHECK-NEXT: str z9, [sp, #15, mul vl] // 16-byte Folded Spill372; CHECK-NEXT: str z8, [sp, #16, mul vl] // 16-byte Folded Spill373; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16374; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16375; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16376; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 32 * VG - 16377; CHECK-NEXT: .cfi_escape 0x10, 0x4c, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x40, 0x1c // $d12 @ cfa - 40 * VG - 16378; CHECK-NEXT: .cfi_escape 0x10, 0x4d, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x40, 0x1c // $d13 @ cfa - 48 * VG - 16379; CHECK-NEXT: .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16380; CHECK-NEXT: .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16381; CHECK-NEXT: //APP382; CHECK-NEXT: //NO_APP383; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload384; CHECK-NEXT: ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload385; CHECK-NEXT: ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload386; CHECK-NEXT: ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload387; CHECK-NEXT: ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload388; CHECK-NEXT: ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload389; CHECK-NEXT: ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload390; CHECK-NEXT: ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload391; CHECK-NEXT: ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload392; CHECK-NEXT: ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload393; CHECK-NEXT: ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload394; CHECK-NEXT: ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload395; CHECK-NEXT: ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload396; CHECK-NEXT: ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload397; CHECK-NEXT: ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload398; CHECK-NEXT: ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload399; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload400; CHECK-NEXT: addvl sp, sp, #17401; CHECK-NEXT: .cfi_def_cfa wsp, 16402; CHECK-NEXT: .cfi_restore z8403; CHECK-NEXT: .cfi_restore z9404; CHECK-NEXT: .cfi_restore z10405; CHECK-NEXT: .cfi_restore z11406; CHECK-NEXT: .cfi_restore z12407; CHECK-NEXT: .cfi_restore z13408; CHECK-NEXT: .cfi_restore z14409; CHECK-NEXT: .cfi_restore z15410; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload411; CHECK-NEXT: .cfi_def_cfa_offset 0412; CHECK-NEXT: .cfi_restore w29413; CHECK-NEXT: ret414entry:415 call void asm sideeffect "", "~{p8},~{z8},~{z9},~{z10},~{z11},~{z12},~{z13},~{z14},~{z15},~{z16},~{z17},~{z18},~{z19},~{z20},~{z21},~{z22},~{z23}" ()416 ret void417}418 419; A SVE vector and a 16-byte fixed size object.420define void @sve_1_vector_16_arr(ptr %out) #0 {421; CHECK-LABEL: sve_1_vector_16_arr:422; CHECK: // %bb.0: // %entry423; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill424; CHECK-NEXT: .cfi_def_cfa_offset 16425; CHECK-NEXT: .cfi_offset w29, -16426; CHECK-NEXT: sub sp, sp, #16427; CHECK-NEXT: .cfi_def_cfa_offset 32428; CHECK-NEXT: addvl sp, sp, #-1429; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 32 + 8 * VG430; CHECK-NEXT: addvl sp, sp, #1431; CHECK-NEXT: .cfi_def_cfa wsp, 32432; CHECK-NEXT: add sp, sp, #16433; CHECK-NEXT: .cfi_def_cfa_offset 16434; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload435; CHECK-NEXT: .cfi_def_cfa_offset 0436; CHECK-NEXT: .cfi_restore w29437; CHECK-NEXT: ret438entry:439 %vec = alloca <vscale x 4 x float>, align 16440 %arr = alloca i8, i64 16, align 1441 ret void442}443 444; A large SVE stack object and a large stack slot, both of which need probing.445; TODO: This could be optimised by combining the fixed-size offset into the446; loop.447define void @sve_1_vector_4096_arr(ptr %out) #0 {448; CHECK-LABEL: sve_1_vector_4096_arr:449; CHECK: // %bb.0: // %entry450; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill451; CHECK-NEXT: .cfi_def_cfa_offset 16452; CHECK-NEXT: .cfi_offset w29, -16453; CHECK-NEXT: sub x9, sp, #3, lsl #12 // =12288454; CHECK-NEXT: .cfi_def_cfa w9, 12304455; CHECK-NEXT: addvl x9, x9, #-32456; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x79, 0x90, 0xe0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x02, 0x1e, 0x22 // $x9 + 12304 + 256 * VG457; CHECK-NEXT: addvl x9, x9, #-32458; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x79, 0x90, 0xe0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x04, 0x1e, 0x22 // $x9 + 12304 + 512 * VG459; CHECK-NEXT: .LBB11_1: // %entry460; CHECK-NEXT: // =>This Inner Loop Header: Depth=1461; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096462; CHECK-NEXT: cmp sp, x9463; CHECK-NEXT: b.le .LBB11_3464; CHECK-NEXT: // %bb.2: // %entry465; CHECK-NEXT: // in Loop: Header=BB11_1 Depth=1466; CHECK-NEXT: str xzr, [sp]467; CHECK-NEXT: b .LBB11_1468; CHECK-NEXT: .LBB11_3: // %entry469; CHECK-NEXT: mov sp, x9470; CHECK-NEXT: ldr xzr, [sp]471; CHECK-NEXT: .cfi_def_cfa_register wsp472; CHECK-NEXT: addvl sp, sp, #31473; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x90, 0xe0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x88, 0x02, 0x1e, 0x22 // sp + 12304 + 264 * VG474; CHECK-NEXT: addvl sp, sp, #31475; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x90, 0xe0, 0x00, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 12304 + 16 * VG476; CHECK-NEXT: addvl sp, sp, #2477; CHECK-NEXT: .cfi_def_cfa wsp, 12304478; CHECK-NEXT: add sp, sp, #3, lsl #12 // =12288479; CHECK-NEXT: .cfi_def_cfa_offset 16480; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload481; CHECK-NEXT: .cfi_def_cfa_offset 0482; CHECK-NEXT: .cfi_restore w29483; CHECK-NEXT: ret484entry:485 %vec = alloca <vscale x 256 x float>, align 16486 %arr = alloca i8, i64 12288, align 1487 ret void488}489 490; Not tested: SVE stack objects with alignment >16 bytes, which isn't currently491; supported even without stack-probing.492 493; An SVE vector, and a 16-byte fixed size object, which494; has a large alignment requirement.495define void @sve_1_vector_16_arr_align_8192(ptr %out) #0 {496; CHECK-LABEL: sve_1_vector_16_arr_align_8192:497; CHECK: // %bb.0: // %entry498; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill499; CHECK-NEXT: .cfi_def_cfa_offset 16500; CHECK-NEXT: mov x29, sp501; CHECK-NEXT: .cfi_def_cfa w29, 16502; CHECK-NEXT: .cfi_offset w30, -8503; CHECK-NEXT: .cfi_offset w29, -16504; CHECK-NEXT: sub x9, sp, #1, lsl #12 // =4096505; CHECK-NEXT: sub x9, x9, #4080506; CHECK-NEXT: addvl x9, x9, #-1507; CHECK-NEXT: and x9, x9, #0xffffffffffffe000508; CHECK-NEXT: .LBB12_1: // %entry509; CHECK-NEXT: // =>This Inner Loop Header: Depth=1510; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096511; CHECK-NEXT: cmp sp, x9512; CHECK-NEXT: b.le .LBB12_3513; CHECK-NEXT: // %bb.2: // %entry514; CHECK-NEXT: // in Loop: Header=BB12_1 Depth=1515; CHECK-NEXT: str xzr, [sp]516; CHECK-NEXT: b .LBB12_1517; CHECK-NEXT: .LBB12_3: // %entry518; CHECK-NEXT: mov sp, x9519; CHECK-NEXT: ldr xzr, [sp]520; CHECK-NEXT: mov sp, x29521; CHECK-NEXT: .cfi_def_cfa wsp, 16522; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload523; CHECK-NEXT: .cfi_def_cfa_offset 0524; CHECK-NEXT: .cfi_restore w30525; CHECK-NEXT: .cfi_restore w29526; CHECK-NEXT: ret527entry:528 %vec = alloca <vscale x 4 x float>, align 16529 %arr = alloca i8, i64 16, align 8192530 ret void531}532 533; With 64k guard pages, we can allocate bigger SVE space without a probing loop.534define void @sve_1024_64k_guard(ptr %out) #0 "stack-probe-size"="65536" {535; CHECK-LABEL: sve_1024_64k_guard:536; CHECK: // %bb.0: // %entry537; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill538; CHECK-NEXT: .cfi_def_cfa_offset 16539; CHECK-NEXT: .cfi_offset w29, -16540; CHECK-NEXT: addvl sp, sp, #-32541; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x02, 0x1e, 0x22 // sp + 16 + 256 * VG542; CHECK-NEXT: addvl sp, sp, #-32543; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x04, 0x1e, 0x22 // sp + 16 + 512 * VG544; CHECK-NEXT: addvl sp, sp, #-32545; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x06, 0x1e, 0x22 // sp + 16 + 768 * VG546; CHECK-NEXT: addvl sp, sp, #-32547; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x08, 0x1e, 0x22 // sp + 16 + 1024 * VG548; CHECK-NEXT: addvl sp, sp, #-32549; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x0a, 0x1e, 0x22 // sp + 16 + 1280 * VG550; CHECK-NEXT: addvl sp, sp, #-32551; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x0c, 0x1e, 0x22 // sp + 16 + 1536 * VG552; CHECK-NEXT: addvl sp, sp, #-32553; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x0e, 0x1e, 0x22 // sp + 16 + 1792 * VG554; CHECK-NEXT: addvl sp, sp, #-32555; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x10, 0x1e, 0x22 // sp + 16 + 2048 * VG556; CHECK-NEXT: str xzr, [sp]557; CHECK-NEXT: addvl sp, sp, #31558; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x88, 0x0e, 0x1e, 0x22 // sp + 16 + 1800 * VG559; CHECK-NEXT: addvl sp, sp, #31560; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x0c, 0x1e, 0x22 // sp + 16 + 1552 * VG561; CHECK-NEXT: addvl sp, sp, #31562; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x0a, 0x1e, 0x22 // sp + 16 + 1304 * VG563; CHECK-NEXT: addvl sp, sp, #31564; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xa0, 0x08, 0x1e, 0x22 // sp + 16 + 1056 * VG565; CHECK-NEXT: addvl sp, sp, #31566; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xa8, 0x06, 0x1e, 0x22 // sp + 16 + 808 * VG567; CHECK-NEXT: addvl sp, sp, #31568; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xb0, 0x04, 0x1e, 0x22 // sp + 16 + 560 * VG569; CHECK-NEXT: addvl sp, sp, #31570; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xb8, 0x02, 0x1e, 0x22 // sp + 16 + 312 * VG571; CHECK-NEXT: addvl sp, sp, #31572; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG573; CHECK-NEXT: addvl sp, sp, #8574; CHECK-NEXT: .cfi_def_cfa wsp, 16575; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload576; CHECK-NEXT: .cfi_def_cfa_offset 0577; CHECK-NEXT: .cfi_restore w29578; CHECK-NEXT: ret579entry:580 %vec = alloca <vscale x 1024 x float>, align 16581 ret void582}583 584define void @sve_1028_64k_guard(ptr %out) #0 "stack-probe-size"="65536" {585; CHECK-LABEL: sve_1028_64k_guard:586; CHECK: // %bb.0: // %entry587; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill588; CHECK-NEXT: .cfi_def_cfa_offset 16589; CHECK-NEXT: .cfi_offset w29, -16590; CHECK-NEXT: addvl x9, sp, #-32591; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x02, 0x1e, 0x22 // $x9 + 16 + 256 * VG592; CHECK-NEXT: addvl x9, x9, #-32593; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x04, 0x1e, 0x22 // $x9 + 16 + 512 * VG594; CHECK-NEXT: addvl x9, x9, #-32595; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x06, 0x1e, 0x22 // $x9 + 16 + 768 * VG596; CHECK-NEXT: addvl x9, x9, #-32597; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x08, 0x1e, 0x22 // $x9 + 16 + 1024 * VG598; CHECK-NEXT: addvl x9, x9, #-32599; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x0a, 0x1e, 0x22 // $x9 + 16 + 1280 * VG600; CHECK-NEXT: addvl x9, x9, #-32601; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x0c, 0x1e, 0x22 // $x9 + 16 + 1536 * VG602; CHECK-NEXT: addvl x9, x9, #-32603; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x0e, 0x1e, 0x22 // $x9 + 16 + 1792 * VG604; CHECK-NEXT: addvl x9, x9, #-32605; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x80, 0x10, 0x1e, 0x22 // $x9 + 16 + 2048 * VG606; CHECK-NEXT: addvl x9, x9, #-1607; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x79, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x88, 0x10, 0x1e, 0x22 // $x9 + 16 + 2056 * VG608; CHECK-NEXT: .LBB14_1: // %entry609; CHECK-NEXT: // =>This Inner Loop Header: Depth=1610; CHECK-NEXT: sub sp, sp, #16, lsl #12 // =65536611; CHECK-NEXT: cmp sp, x9612; CHECK-NEXT: b.le .LBB14_3613; CHECK-NEXT: // %bb.2: // %entry614; CHECK-NEXT: // in Loop: Header=BB14_1 Depth=1615; CHECK-NEXT: str xzr, [sp]616; CHECK-NEXT: b .LBB14_1617; CHECK-NEXT: .LBB14_3: // %entry618; CHECK-NEXT: mov sp, x9619; CHECK-NEXT: ldr xzr, [sp]620; CHECK-NEXT: .cfi_def_cfa_register wsp621; CHECK-NEXT: addvl sp, sp, #31622; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x0e, 0x1e, 0x22 // sp + 16 + 1808 * VG623; CHECK-NEXT: addvl sp, sp, #31624; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x0c, 0x1e, 0x22 // sp + 16 + 1560 * VG625; CHECK-NEXT: addvl sp, sp, #31626; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xa0, 0x0a, 0x1e, 0x22 // sp + 16 + 1312 * VG627; CHECK-NEXT: addvl sp, sp, #31628; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xa8, 0x08, 0x1e, 0x22 // sp + 16 + 1064 * VG629; CHECK-NEXT: addvl sp, sp, #31630; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xb0, 0x06, 0x1e, 0x22 // sp + 16 + 816 * VG631; CHECK-NEXT: addvl sp, sp, #31632; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xb8, 0x04, 0x1e, 0x22 // sp + 16 + 568 * VG633; CHECK-NEXT: addvl sp, sp, #31634; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x02, 0x1e, 0x22 // sp + 16 + 320 * VG635; CHECK-NEXT: addvl sp, sp, #31636; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc8, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG637; CHECK-NEXT: addvl sp, sp, #9638; CHECK-NEXT: .cfi_def_cfa wsp, 16639; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload640; CHECK-NEXT: .cfi_def_cfa_offset 0641; CHECK-NEXT: .cfi_restore w29642; CHECK-NEXT: ret643entry:644 %vec = alloca <vscale x 1024 x float>, align 16645 %vec1 = alloca <vscale x 4 x float>, align 16646 ret void647}648 649; With 5 SVE vectors of stack space the unprobed area650; at the top of the stack can exceed 1024 bytes (5 x 256 == 1280),651; hence we need to issue a probe.652define void @sve_5_vector(ptr %out) #0 {653; CHECK-LABEL: sve_5_vector:654; CHECK: // %bb.0: // %entry655; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill656; CHECK-NEXT: .cfi_def_cfa_offset 16657; CHECK-NEXT: .cfi_offset w29, -16658; CHECK-NEXT: addvl sp, sp, #-5659; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG660; CHECK-NEXT: str xzr, [sp]661; CHECK-NEXT: addvl sp, sp, #5662; CHECK-NEXT: .cfi_def_cfa wsp, 16663; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload664; CHECK-NEXT: .cfi_def_cfa_offset 0665; CHECK-NEXT: .cfi_restore w29666; CHECK-NEXT: ret667entry:668 %vec1 = alloca <vscale x 4 x float>, align 16669 %vec2 = alloca <vscale x 4 x float>, align 16670 %vec3 = alloca <vscale x 4 x float>, align 16671 %vec4 = alloca <vscale x 4 x float>, align 16672 %vec5 = alloca <vscale x 4 x float>, align 16673 ret void674}675 676; Test with a 14 scalable bytes (so up to 14 * 16 = 224) of unprobed677; are bellow the save location of `p9`.678define void @sve_unprobed_area(<vscale x 4 x float> %a, i32 %n) #0 {679; CHECK-LABEL: sve_unprobed_area:680; CHECK: // %bb.0: // %entry681; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill682; CHECK-NEXT: .cfi_def_cfa_offset 16683; CHECK-NEXT: .cfi_offset w29, -16684; CHECK-NEXT: addvl sp, sp, #-4685; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG686; CHECK-NEXT: str xzr, [sp]687; CHECK-NEXT: str p9, [sp, #7, mul vl] // 2-byte Spill688; CHECK-NEXT: str z10, [sp, #1, mul vl] // 16-byte Folded Spill689; CHECK-NEXT: str z9, [sp, #2, mul vl] // 16-byte Folded Spill690; CHECK-NEXT: str z8, [sp, #3, mul vl] // 16-byte Folded Spill691; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16692; CHECK-NEXT: .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16693; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16694; CHECK-NEXT: addvl sp, sp, #-4695; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc0, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG696; CHECK-NEXT: //APP697; CHECK-NEXT: //NO_APP698; CHECK-NEXT: addvl sp, sp, #4699; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG700; CHECK-NEXT: ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload701; CHECK-NEXT: ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload702; CHECK-NEXT: ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload703; CHECK-NEXT: ldr p9, [sp, #7, mul vl] // 2-byte Reload704; CHECK-NEXT: addvl sp, sp, #4705; CHECK-NEXT: .cfi_def_cfa wsp, 16706; CHECK-NEXT: .cfi_restore z8707; CHECK-NEXT: .cfi_restore z9708; CHECK-NEXT: .cfi_restore z10709; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload710; CHECK-NEXT: .cfi_def_cfa_offset 0711; CHECK-NEXT: .cfi_restore w29712; CHECK-NEXT: ret713entry:714 call void asm sideeffect "", "~{z8},~{z9},~{z10},~{p9}" ()715 716 %v0 = alloca <vscale x 4 x float>, align 16717 %v1 = alloca <vscale x 4 x float>, align 16718 %v2 = alloca <vscale x 4 x float>, align 16719 %v3 = alloca <vscale x 4 x float>, align 16720 721 ret void722}723 724attributes #0 = { uwtable(async) "probe-stack"="inline-asm" "frame-pointer"="none" "target-features"="+sve" }725