286 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK3; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -verify-machineinstrs -aarch64-lower-to-sme-routines=false < %s | FileCheck %s -check-prefixes=CHECK-NO-SME-ROUTINES4; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -mattr=+mops -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK-MOPS5 6@dst = global [512 x i8] zeroinitializer, align 17@src = global [512 x i8] zeroinitializer, align 18 9define void @se_memcpy(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {10; CHECK-LABEL: se_memcpy:11; CHECK: // %bb.0: // %entry12; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill13; CHECK-NEXT: mov x2, x014; CHECK-NEXT: adrp x0, :got:dst15; CHECK-NEXT: adrp x1, :got:src16; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]17; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]18; CHECK-NEXT: bl __arm_sc_memcpy19; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload20; CHECK-NEXT: ret21;22; CHECK-NO-SME-ROUTINES-LABEL: se_memcpy:23; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry24; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill25; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill26; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x027; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst28; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill29; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src30; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill31; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill32; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]33; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]34; CHECK-NO-SME-ROUTINES-NEXT: smstop sm35; CHECK-NO-SME-ROUTINES-NEXT: bl memcpy36; CHECK-NO-SME-ROUTINES-NEXT: smstart sm37; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload38; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload39; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload40; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload41; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload42; CHECK-NO-SME-ROUTINES-NEXT: ret43;44; CHECK-MOPS-LABEL: se_memcpy:45; CHECK-MOPS: // %bb.0: // %entry46; CHECK-MOPS-NEXT: adrp x8, :got:src47; CHECK-MOPS-NEXT: adrp x9, :got:dst48; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]49; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]50; CHECK-MOPS-NEXT: cpyfp [x9]!, [x8]!, x0!51; CHECK-MOPS-NEXT: cpyfm [x9]!, [x8]!, x0!52; CHECK-MOPS-NEXT: cpyfe [x9]!, [x8]!, x0!53; CHECK-MOPS-NEXT: ret54entry:55 tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)56 ret void57}58 59define void @se_memset(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {60; CHECK-LABEL: se_memset:61; CHECK: // %bb.0: // %entry62; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill63; CHECK-NEXT: mov x2, x064; CHECK-NEXT: adrp x0, :got:dst65; CHECK-NEXT: mov w1, #2 // =0x266; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]67; CHECK-NEXT: bl __arm_sc_memset68; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload69; CHECK-NEXT: ret70;71; CHECK-NO-SME-ROUTINES-LABEL: se_memset:72; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry73; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill74; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill75; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x076; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst77; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill78; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill79; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill80; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]81; CHECK-NO-SME-ROUTINES-NEXT: smstop sm82; CHECK-NO-SME-ROUTINES-NEXT: mov w1, #2 // =0x283; CHECK-NO-SME-ROUTINES-NEXT: bl memset84; CHECK-NO-SME-ROUTINES-NEXT: smstart sm85; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload86; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload87; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload88; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload89; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload90; CHECK-NO-SME-ROUTINES-NEXT: ret91;92; CHECK-MOPS-LABEL: se_memset:93; CHECK-MOPS: // %bb.0: // %entry94; CHECK-MOPS-NEXT: adrp x8, :got:dst95; CHECK-MOPS-NEXT: mov w9, #2 // =0x296; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:dst]97; CHECK-MOPS-NEXT: setp [x8]!, x0!, x998; CHECK-MOPS-NEXT: setm [x8]!, x0!, x999; CHECK-MOPS-NEXT: sete [x8]!, x0!, x9100; CHECK-MOPS-NEXT: ret101entry:102 tail call void @llvm.memset.p0.i64(ptr align 1 @dst, i8 2, i64 %n, i1 false)103 ret void104}105 106define void @se_memmove(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {107; CHECK-LABEL: se_memmove:108; CHECK: // %bb.0: // %entry109; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill110; CHECK-NEXT: mov x2, x0111; CHECK-NEXT: adrp x0, :got:dst112; CHECK-NEXT: adrp x1, :got:src113; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]114; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]115; CHECK-NEXT: bl __arm_sc_memmove116; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload117; CHECK-NEXT: ret118;119; CHECK-NO-SME-ROUTINES-LABEL: se_memmove:120; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry121; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill122; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill123; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0124; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst125; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill126; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src127; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill128; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill129; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]130; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]131; CHECK-NO-SME-ROUTINES-NEXT: smstop sm132; CHECK-NO-SME-ROUTINES-NEXT: bl memmove133; CHECK-NO-SME-ROUTINES-NEXT: smstart sm134; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload135; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload136; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload137; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload138; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload139; CHECK-NO-SME-ROUTINES-NEXT: ret140;141; CHECK-MOPS-LABEL: se_memmove:142; CHECK-MOPS: // %bb.0: // %entry143; CHECK-MOPS-NEXT: adrp x8, :got:src144; CHECK-MOPS-NEXT: adrp x9, :got:dst145; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]146; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]147; CHECK-MOPS-NEXT: cpyp [x9]!, [x8]!, x0!148; CHECK-MOPS-NEXT: cpym [x9]!, [x8]!, x0!149; CHECK-MOPS-NEXT: cpye [x9]!, [x8]!, x0!150; CHECK-MOPS-NEXT: ret151entry:152 tail call void @llvm.memmove.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)153 ret void154}155 156define void @sc_memcpy(i64 noundef %n) "aarch64_pstate_sm_compatible" nounwind {157; CHECK-LABEL: sc_memcpy:158; CHECK: // %bb.0: // %entry159; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill160; CHECK-NEXT: mov x2, x0161; CHECK-NEXT: adrp x0, :got:dst162; CHECK-NEXT: adrp x1, :got:src163; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]164; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]165; CHECK-NEXT: bl __arm_sc_memcpy166; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload167; CHECK-NEXT: ret168;169; CHECK-NO-SME-ROUTINES-LABEL: sc_memcpy:170; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry171; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill172; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0173; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst174; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src175; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill176; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill177; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill178; CHECK-NO-SME-ROUTINES-NEXT: stp x30, x19, [sp, #64] // 16-byte Folded Spill179; CHECK-NO-SME-ROUTINES-NEXT: mrs x19, SVCR180; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]181; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]182; CHECK-NO-SME-ROUTINES-NEXT: tbz w19, #0, .LBB3_2183; CHECK-NO-SME-ROUTINES-NEXT: // %bb.1: // %entry184; CHECK-NO-SME-ROUTINES-NEXT: smstop sm185; CHECK-NO-SME-ROUTINES-NEXT: .LBB3_2: // %entry186; CHECK-NO-SME-ROUTINES-NEXT: bl memcpy187; CHECK-NO-SME-ROUTINES-NEXT: tbz w19, #0, .LBB3_4188; CHECK-NO-SME-ROUTINES-NEXT: // %bb.3: // %entry189; CHECK-NO-SME-ROUTINES-NEXT: smstart sm190; CHECK-NO-SME-ROUTINES-NEXT: .LBB3_4: // %entry191; CHECK-NO-SME-ROUTINES-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload192; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload193; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload194; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload195; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload196; CHECK-NO-SME-ROUTINES-NEXT: ret197;198; CHECK-MOPS-LABEL: sc_memcpy:199; CHECK-MOPS: // %bb.0: // %entry200; CHECK-MOPS-NEXT: adrp x8, :got:src201; CHECK-MOPS-NEXT: adrp x9, :got:dst202; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]203; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]204; CHECK-MOPS-NEXT: cpyfp [x9]!, [x8]!, x0!205; CHECK-MOPS-NEXT: cpyfm [x9]!, [x8]!, x0!206; CHECK-MOPS-NEXT: cpyfe [x9]!, [x8]!, x0!207; CHECK-MOPS-NEXT: ret208entry:209 tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)210 ret void211}212 213define void @sb_memcpy(i64 noundef %n) "aarch64_pstate_sm_body" nounwind {214; CHECK-LABEL: sb_memcpy:215; CHECK: // %bb.0: // %entry216; CHECK-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill217; CHECK-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill218; CHECK-NEXT: mov x2, x0219; CHECK-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill220; CHECK-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill221; CHECK-NEXT: str x30, [sp, #64] // 8-byte Spill222; CHECK-NEXT: smstart sm223; CHECK-NEXT: adrp x0, :got:dst224; CHECK-NEXT: adrp x1, :got:src225; CHECK-NEXT: ldr x0, [x0, :got_lo12:dst]226; CHECK-NEXT: ldr x1, [x1, :got_lo12:src]227; CHECK-NEXT: bl __arm_sc_memcpy228; CHECK-NEXT: smstop sm229; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload230; CHECK-NEXT: ldr x30, [sp, #64] // 8-byte Reload231; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload232; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload233; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload234; CHECK-NEXT: ret235;236; CHECK-NO-SME-ROUTINES-LABEL: sb_memcpy:237; CHECK-NO-SME-ROUTINES: // %bb.0: // %entry238; CHECK-NO-SME-ROUTINES-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill239; CHECK-NO-SME-ROUTINES-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill240; CHECK-NO-SME-ROUTINES-NEXT: mov x2, x0241; CHECK-NO-SME-ROUTINES-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill242; CHECK-NO-SME-ROUTINES-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill243; CHECK-NO-SME-ROUTINES-NEXT: str x30, [sp, #64] // 8-byte Spill244; CHECK-NO-SME-ROUTINES-NEXT: smstart sm245; CHECK-NO-SME-ROUTINES-NEXT: adrp x0, :got:dst246; CHECK-NO-SME-ROUTINES-NEXT: adrp x1, :got:src247; CHECK-NO-SME-ROUTINES-NEXT: ldr x0, [x0, :got_lo12:dst]248; CHECK-NO-SME-ROUTINES-NEXT: ldr x1, [x1, :got_lo12:src]249; CHECK-NO-SME-ROUTINES-NEXT: smstop sm250; CHECK-NO-SME-ROUTINES-NEXT: bl memcpy251; CHECK-NO-SME-ROUTINES-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload252; CHECK-NO-SME-ROUTINES-NEXT: ldr x30, [sp, #64] // 8-byte Reload253; CHECK-NO-SME-ROUTINES-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload254; CHECK-NO-SME-ROUTINES-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload255; CHECK-NO-SME-ROUTINES-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload256; CHECK-NO-SME-ROUTINES-NEXT: ret257;258; CHECK-MOPS-LABEL: sb_memcpy:259; CHECK-MOPS: // %bb.0: // %entry260; CHECK-MOPS-NEXT: stp d15, d14, [sp, #-64]! // 16-byte Folded Spill261; CHECK-MOPS-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill262; CHECK-MOPS-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill263; CHECK-MOPS-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill264; CHECK-MOPS-NEXT: smstart sm265; CHECK-MOPS-NEXT: adrp x8, :got:src266; CHECK-MOPS-NEXT: adrp x9, :got:dst267; CHECK-MOPS-NEXT: ldr x8, [x8, :got_lo12:src]268; CHECK-MOPS-NEXT: ldr x9, [x9, :got_lo12:dst]269; CHECK-MOPS-NEXT: cpyfp [x9]!, [x8]!, x0!270; CHECK-MOPS-NEXT: cpyfm [x9]!, [x8]!, x0!271; CHECK-MOPS-NEXT: cpyfe [x9]!, [x8]!, x0!272; CHECK-MOPS-NEXT: smstop sm273; CHECK-MOPS-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload274; CHECK-MOPS-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload275; CHECK-MOPS-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload276; CHECK-MOPS-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload277; CHECK-MOPS-NEXT: ret278entry:279 tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)280 ret void281}282 283declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)284declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)285declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)286