880 lines · c
1//===- AArch64InstrInfo.h - AArch64 Instruction Information -----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the AArch64 implementation of the TargetInstrInfo class.10//11//===----------------------------------------------------------------------===//12 13#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H14#define LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H15 16#include "AArch64.h"17#include "AArch64RegisterInfo.h"18#include "llvm/CodeGen/TargetInstrInfo.h"19#include "llvm/Support/TypeSize.h"20#include <optional>21 22#define GET_INSTRINFO_HEADER23#include "AArch64GenInstrInfo.inc"24 25namespace llvm {26 27class AArch64Subtarget;28 29static const MachineMemOperand::Flags MOSuppressPair =30 MachineMemOperand::MOTargetFlag1;31static const MachineMemOperand::Flags MOStridedAccess =32 MachineMemOperand::MOTargetFlag2;33 34#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"35 36// AArch64 MachineCombiner patterns37enum AArch64MachineCombinerPattern : unsigned {38 // These are patterns used to reduce the length of dependence chain.39 SUBADD_OP1 = MachineCombinerPattern::TARGET_PATTERN_START,40 SUBADD_OP2,41 42 // These are multiply-add patterns matched by the AArch64 machine combiner.43 MULADDW_OP1,44 MULADDW_OP2,45 MULSUBW_OP1,46 MULSUBW_OP2,47 MULADDWI_OP1,48 MULSUBWI_OP1,49 MULADDX_OP1,50 MULADDX_OP2,51 MULSUBX_OP1,52 MULSUBX_OP2,53 MULADDXI_OP1,54 MULSUBXI_OP1,55 // NEON integers vectors56 MULADDv8i8_OP1,57 MULADDv8i8_OP2,58 MULADDv16i8_OP1,59 MULADDv16i8_OP2,60 MULADDv4i16_OP1,61 MULADDv4i16_OP2,62 MULADDv8i16_OP1,63 MULADDv8i16_OP2,64 MULADDv2i32_OP1,65 MULADDv2i32_OP2,66 MULADDv4i32_OP1,67 MULADDv4i32_OP2,68 69 MULSUBv8i8_OP1,70 MULSUBv8i8_OP2,71 MULSUBv16i8_OP1,72 MULSUBv16i8_OP2,73 MULSUBv4i16_OP1,74 MULSUBv4i16_OP2,75 MULSUBv8i16_OP1,76 MULSUBv8i16_OP2,77 MULSUBv2i32_OP1,78 MULSUBv2i32_OP2,79 MULSUBv4i32_OP1,80 MULSUBv4i32_OP2,81 82 MULADDv4i16_indexed_OP1,83 MULADDv4i16_indexed_OP2,84 MULADDv8i16_indexed_OP1,85 MULADDv8i16_indexed_OP2,86 MULADDv2i32_indexed_OP1,87 MULADDv2i32_indexed_OP2,88 MULADDv4i32_indexed_OP1,89 MULADDv4i32_indexed_OP2,90 91 MULSUBv4i16_indexed_OP1,92 MULSUBv4i16_indexed_OP2,93 MULSUBv8i16_indexed_OP1,94 MULSUBv8i16_indexed_OP2,95 MULSUBv2i32_indexed_OP1,96 MULSUBv2i32_indexed_OP2,97 MULSUBv4i32_indexed_OP1,98 MULSUBv4i32_indexed_OP2,99 100 // Floating Point101 FMULADDH_OP1,102 FMULADDH_OP2,103 FMULSUBH_OP1,104 FMULSUBH_OP2,105 FMULADDS_OP1,106 FMULADDS_OP2,107 FMULSUBS_OP1,108 FMULSUBS_OP2,109 FMULADDD_OP1,110 FMULADDD_OP2,111 FMULSUBD_OP1,112 FMULSUBD_OP2,113 FNMULSUBH_OP1,114 FNMULSUBS_OP1,115 FNMULSUBD_OP1,116 FMLAv1i32_indexed_OP1,117 FMLAv1i32_indexed_OP2,118 FMLAv1i64_indexed_OP1,119 FMLAv1i64_indexed_OP2,120 FMLAv4f16_OP1,121 FMLAv4f16_OP2,122 FMLAv8f16_OP1,123 FMLAv8f16_OP2,124 FMLAv2f32_OP2,125 FMLAv2f32_OP1,126 FMLAv2f64_OP1,127 FMLAv2f64_OP2,128 FMLAv4i16_indexed_OP1,129 FMLAv4i16_indexed_OP2,130 FMLAv8i16_indexed_OP1,131 FMLAv8i16_indexed_OP2,132 FMLAv2i32_indexed_OP1,133 FMLAv2i32_indexed_OP2,134 FMLAv2i64_indexed_OP1,135 FMLAv2i64_indexed_OP2,136 FMLAv4f32_OP1,137 FMLAv4f32_OP2,138 FMLAv4i32_indexed_OP1,139 FMLAv4i32_indexed_OP2,140 FMLSv1i32_indexed_OP2,141 FMLSv1i64_indexed_OP2,142 FMLSv4f16_OP1,143 FMLSv4f16_OP2,144 FMLSv8f16_OP1,145 FMLSv8f16_OP2,146 FMLSv2f32_OP1,147 FMLSv2f32_OP2,148 FMLSv2f64_OP1,149 FMLSv2f64_OP2,150 FMLSv4i16_indexed_OP1,151 FMLSv4i16_indexed_OP2,152 FMLSv8i16_indexed_OP1,153 FMLSv8i16_indexed_OP2,154 FMLSv2i32_indexed_OP1,155 FMLSv2i32_indexed_OP2,156 FMLSv2i64_indexed_OP1,157 FMLSv2i64_indexed_OP2,158 FMLSv4f32_OP1,159 FMLSv4f32_OP2,160 FMLSv4i32_indexed_OP1,161 FMLSv4i32_indexed_OP2,162 163 FMULv2i32_indexed_OP1,164 FMULv2i32_indexed_OP2,165 FMULv2i64_indexed_OP1,166 FMULv2i64_indexed_OP2,167 FMULv4i16_indexed_OP1,168 FMULv4i16_indexed_OP2,169 FMULv4i32_indexed_OP1,170 FMULv4i32_indexed_OP2,171 FMULv8i16_indexed_OP1,172 FMULv8i16_indexed_OP2,173 174 FNMADD,175 176 GATHER_LANE_i32,177 GATHER_LANE_i16,178 GATHER_LANE_i8179};180class AArch64InstrInfo final : public AArch64GenInstrInfo {181 const AArch64RegisterInfo RI;182 const AArch64Subtarget &Subtarget;183 184public:185 explicit AArch64InstrInfo(const AArch64Subtarget &STI);186 187 /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info. As188 /// such, whenever a client has an instance of instruction info, it should189 /// always be able to get register info as well (through this method).190 const AArch64RegisterInfo &getRegisterInfo() const { return RI; }191 192 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;193 194 bool isAsCheapAsAMove(const MachineInstr &MI) const override;195 196 bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,197 Register &DstReg, unsigned &SubIdx) const override;198 199 bool200 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,201 const MachineInstr &MIb) const override;202 203 Register isLoadFromStackSlot(const MachineInstr &MI,204 int &FrameIndex) const override;205 Register isStoreToStackSlot(const MachineInstr &MI,206 int &FrameIndex) const override;207 208 /// Check for post-frame ptr elimination stack locations as well. This uses a209 /// heuristic so it isn't reliable for correctness.210 Register isStoreToStackSlotPostFE(const MachineInstr &MI,211 int &FrameIndex) const override;212 /// Check for post-frame ptr elimination stack locations as well. This uses a213 /// heuristic so it isn't reliable for correctness.214 Register isLoadFromStackSlotPostFE(const MachineInstr &MI,215 int &FrameIndex) const override;216 217 /// Does this instruction set its full destination register to zero?218 static bool isGPRZero(const MachineInstr &MI);219 220 /// Does this instruction rename a GPR without modifying bits?221 static bool isGPRCopy(const MachineInstr &MI);222 223 /// Does this instruction rename an FPR without modifying bits?224 static bool isFPRCopy(const MachineInstr &MI);225 226 /// Return true if pairing the given load or store is hinted to be227 /// unprofitable.228 static bool isLdStPairSuppressed(const MachineInstr &MI);229 230 /// Return true if the given load or store is a strided memory access.231 static bool isStridedAccess(const MachineInstr &MI);232 233 /// Return true if it has an unscaled load/store offset.234 static bool hasUnscaledLdStOffset(unsigned Opc);235 static bool hasUnscaledLdStOffset(MachineInstr &MI) {236 return hasUnscaledLdStOffset(MI.getOpcode());237 }238 239 /// Returns the unscaled load/store for the scaled load/store opcode,240 /// if there is a corresponding unscaled variant available.241 static std::optional<unsigned> getUnscaledLdSt(unsigned Opc);242 243 /// Scaling factor for (scaled or unscaled) load or store.244 static int getMemScale(unsigned Opc);245 static int getMemScale(const MachineInstr &MI) {246 return getMemScale(MI.getOpcode());247 }248 249 /// Returns whether the instruction is a pre-indexed load.250 static bool isPreLd(const MachineInstr &MI);251 252 /// Returns whether the instruction is a pre-indexed store.253 static bool isPreSt(const MachineInstr &MI);254 255 /// Returns whether the instruction is a pre-indexed load/store.256 static bool isPreLdSt(const MachineInstr &MI);257 258 /// Returns whether the instruction is a paired load/store.259 static bool isPairedLdSt(const MachineInstr &MI);260 261 /// Returns the base register operator of a load/store.262 static const MachineOperand &getLdStBaseOp(const MachineInstr &MI);263 264 /// Returns the immediate offset operator of a load/store.265 static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI);266 267 /// Returns whether the physical register is FP or NEON.268 static bool isFpOrNEON(Register Reg);269 270 /// Returns the shift amount operator of a load/store.271 static const MachineOperand &getLdStAmountOp(const MachineInstr &MI);272 273 /// Returns whether the instruction is FP or NEON.274 static bool isFpOrNEON(const MachineInstr &MI);275 276 /// Returns whether the instruction is in H form (16 bit operands)277 static bool isHForm(const MachineInstr &MI);278 279 /// Returns whether the instruction is in Q form (128 bit operands)280 static bool isQForm(const MachineInstr &MI);281 282 /// Returns whether the instruction can be compatible with non-zero BTYPE.283 static bool hasBTISemantics(const MachineInstr &MI);284 285 /// Returns the index for the immediate for a given instruction.286 static unsigned getLoadStoreImmIdx(unsigned Opc);287 288 /// Return true if pairing the given load or store may be paired with another.289 static bool isPairableLdStInst(const MachineInstr &MI);290 291 /// Returns true if MI is one of the TCRETURN* instructions.292 static bool isTailCallReturnInst(const MachineInstr &MI);293 294 /// Return the opcode that set flags when possible. The caller is295 /// responsible for ensuring the opc has a flag setting equivalent.296 static unsigned convertToFlagSettingOpc(unsigned Opc);297 298 /// Return true if this is a load/store that can be potentially paired/merged.299 bool isCandidateToMergeOrPair(const MachineInstr &MI) const;300 301 /// Hint that pairing the given load or store is unprofitable.302 static void suppressLdStPair(MachineInstr &MI);303 304 std::optional<ExtAddrMode>305 getAddrModeFromMemoryOp(const MachineInstr &MemI,306 const TargetRegisterInfo *TRI) const override;307 308 bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,309 const MachineInstr &AddrI,310 ExtAddrMode &AM) const override;311 312 MachineInstr *emitLdStWithAddr(MachineInstr &MemI,313 const ExtAddrMode &AM) const override;314 315 bool getMemOperandsWithOffsetWidth(316 const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,317 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,318 const TargetRegisterInfo *TRI) const override;319 320 /// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`.321 /// This is true for some SVE instructions like ldr/str that have a322 /// 'reg + imm' addressing mode where the immediate is an index to the323 /// scalable vector located at 'reg + imm * vscale x #bytes'.324 bool getMemOperandWithOffsetWidth(const MachineInstr &MI,325 const MachineOperand *&BaseOp,326 int64_t &Offset, bool &OffsetIsScalable,327 TypeSize &Width,328 const TargetRegisterInfo *TRI) const;329 330 /// Return the immediate offset of the base register in a load/store \p LdSt.331 MachineOperand &getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const;332 333 /// Returns true if opcode \p Opc is a memory operation. If it is, set334 /// \p Scale, \p Width, \p MinOffset, and \p MaxOffset accordingly.335 ///336 /// For unscaled instructions, \p Scale is set to 1. All values are in bytes.337 /// MinOffset/MaxOffset are the un-scaled limits of the immediate in the338 /// instruction, the actual offset limit is [MinOffset*Scale,339 /// MaxOffset*Scale].340 static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width,341 int64_t &MinOffset, int64_t &MaxOffset);342 343 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,344 int64_t Offset1, bool OffsetIsScalable1,345 ArrayRef<const MachineOperand *> BaseOps2,346 int64_t Offset2, bool OffsetIsScalable2,347 unsigned ClusterSize,348 unsigned NumBytes) const override;349 350 void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,351 const DebugLoc &DL, MCRegister DestReg,352 MCRegister SrcReg, bool KillSrc, unsigned Opcode,353 llvm::ArrayRef<unsigned> Indices) const;354 void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,355 const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,356 bool KillSrc, unsigned Opcode, unsigned ZeroReg,357 llvm::ArrayRef<unsigned> Indices) const;358 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,359 const DebugLoc &DL, Register DestReg, Register SrcReg,360 bool KillSrc, bool RenamableDest = false,361 bool RenamableSrc = false) const override;362 363 void storeRegToStackSlot(364 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,365 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,366 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;367 368 void loadRegFromStackSlot(369 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,370 Register DestReg, int FrameIndex, const TargetRegisterClass *RC,371 Register VReg,372 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;373 374 // This tells target independent code that it is okay to pass instructions375 // with subreg operands to foldMemoryOperandImpl.376 bool isSubregFoldable() const override { return true; }377 378 using TargetInstrInfo::foldMemoryOperandImpl;379 MachineInstr *380 foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,381 ArrayRef<unsigned> Ops,382 MachineBasicBlock::iterator InsertPt, int FrameIndex,383 LiveIntervals *LIS = nullptr,384 VirtRegMap *VRM = nullptr) const override;385 386 /// \returns true if a branch from an instruction with opcode \p BranchOpc387 /// bytes is capable of jumping to a position \p BrOffset bytes away.388 bool isBranchOffsetInRange(unsigned BranchOpc,389 int64_t BrOffset) const override;390 391 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;392 393 void insertIndirectBranch(MachineBasicBlock &MBB,394 MachineBasicBlock &NewDestBB,395 MachineBasicBlock &RestoreBB, const DebugLoc &DL,396 int64_t BrOffset, RegScavenger *RS) const override;397 398 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,399 MachineBasicBlock *&FBB,400 SmallVectorImpl<MachineOperand> &Cond,401 bool AllowModify = false) const override;402 bool analyzeBranchPredicate(MachineBasicBlock &MBB,403 MachineBranchPredicate &MBP,404 bool AllowModify) const override;405 unsigned removeBranch(MachineBasicBlock &MBB,406 int *BytesRemoved = nullptr) const override;407 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,408 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,409 const DebugLoc &DL,410 int *BytesAdded = nullptr) const override;411 412 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>413 analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;414 415 bool416 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;417 bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,418 Register, Register, Register, int &, int &,419 int &) const override;420 void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,421 const DebugLoc &DL, Register DstReg,422 ArrayRef<MachineOperand> Cond, Register TrueReg,423 Register FalseReg) const override;424 425 void insertNoop(MachineBasicBlock &MBB,426 MachineBasicBlock::iterator MI) const override;427 428 MCInst getNop() const override;429 430 bool isSchedulingBoundary(const MachineInstr &MI,431 const MachineBasicBlock *MBB,432 const MachineFunction &MF) const override;433 434 /// analyzeCompare - For a comparison instruction, return the source registers435 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.436 /// Return true if the comparison instruction can be analyzed.437 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,438 Register &SrcReg2, int64_t &CmpMask,439 int64_t &CmpValue) const override;440 /// optimizeCompareInstr - Convert the instruction supplying the argument to441 /// the comparison into one that sets the zero bit in the flags register.442 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,443 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,444 const MachineRegisterInfo *MRI) const override;445 bool optimizeCondBranch(MachineInstr &MI) const override;446 447 CombinerObjective getCombinerObjective(unsigned Pattern) const override;448 /// Return true when a code sequence can improve throughput. It449 /// should be called only for instructions in loops.450 /// \param Pattern - combiner pattern451 bool isThroughputPattern(unsigned Pattern) const override;452 /// Return true when there is potentially a faster code sequence453 /// for an instruction chain ending in ``Root``. All potential patterns are454 /// listed in the ``Patterns`` array.455 bool getMachineCombinerPatterns(MachineInstr &Root,456 SmallVectorImpl<unsigned> &Patterns,457 bool DoRegPressureReduce) const override;458 /// Return true when Inst is associative and commutative so that it can be459 /// reassociated. If Invert is true, then the inverse of Inst operation must460 /// be checked.461 bool isAssociativeAndCommutative(const MachineInstr &Inst,462 bool Invert) const override;463 464 /// Returns true if \P Opcode is an instruction which performs accumulation465 /// into a destination register.466 bool isAccumulationOpcode(unsigned Opcode) const override;467 468 /// Returns an opcode which defines the accumulator used by \P Opcode.469 unsigned getAccumulationStartOpcode(unsigned Opcode) const override;470 471 unsigned472 getReduceOpcodeForAccumulator(unsigned int AccumulatorOpCode) const override;473 474 /// When getMachineCombinerPatterns() finds patterns, this function475 /// generates the instructions that could replace the original code476 /// sequence477 void genAlternativeCodeSequence(478 MachineInstr &Root, unsigned Pattern,479 SmallVectorImpl<MachineInstr *> &InsInstrs,480 SmallVectorImpl<MachineInstr *> &DelInstrs,481 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const override;482 /// AArch64 supports MachineCombiner.483 bool useMachineCombiner() const override;484 485 bool expandPostRAPseudo(MachineInstr &MI) const override;486 487 std::pair<unsigned, unsigned>488 decomposeMachineOperandsTargetFlags(unsigned TF) const override;489 ArrayRef<std::pair<unsigned, const char *>>490 getSerializableDirectMachineOperandTargetFlags() const override;491 ArrayRef<std::pair<unsigned, const char *>>492 getSerializableBitmaskMachineOperandTargetFlags() const override;493 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>494 getSerializableMachineMemOperandTargetFlags() const override;495 496 bool isFunctionSafeToOutlineFrom(MachineFunction &MF,497 bool OutlineFromLinkOnceODRs) const override;498 std::optional<std::unique_ptr<outliner::OutlinedFunction>>499 getOutliningCandidateInfo(500 const MachineModuleInfo &MMI,501 std::vector<outliner::Candidate> &RepeatedSequenceLocs,502 unsigned MinRepeats) const override;503 void mergeOutliningCandidateAttributes(504 Function &F, std::vector<outliner::Candidate> &Candidates) const override;505 outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,506 MachineBasicBlock::iterator &MIT,507 unsigned Flags) const override;508 SmallVector<509 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>510 getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override;511 void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF,512 const outliner::OutlinedFunction &OF) const override;513 MachineBasicBlock::iterator514 insertOutlinedCall(Module &M, MachineBasicBlock &MBB,515 MachineBasicBlock::iterator &It, MachineFunction &MF,516 outliner::Candidate &C) const override;517 bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;518 519 void buildClearRegister(Register Reg, MachineBasicBlock &MBB,520 MachineBasicBlock::iterator Iter, DebugLoc &DL,521 bool AllowSideEffects = true) const override;522 523 /// Returns the vector element size (B, H, S or D) of an SVE opcode.524 uint64_t getElementSizeForOpcode(unsigned Opc) const;525 /// Returns true if the opcode is for an SVE instruction that sets the526 /// condition codes as if it's results had been fed to a PTEST instruction527 /// along with the same general predicate.528 bool isPTestLikeOpcode(unsigned Opc) const;529 /// Returns true if the opcode is for an SVE WHILE## instruction.530 bool isWhileOpcode(unsigned Opc) const;531 /// Returns true if the instruction has a shift by immediate that can be532 /// executed in one cycle less.533 static bool isFalkorShiftExtFast(const MachineInstr &MI);534 /// Return true if the instructions is a SEH instruction used for unwinding535 /// on Windows.536 static bool isSEHInstruction(const MachineInstr &MI);537 538 std::optional<RegImmPair> isAddImmediate(const MachineInstr &MI,539 Register Reg) const override;540 541 bool isFunctionSafeToSplit(const MachineFunction &MF) const override;542 543 bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override;544 545 std::optional<ParamLoadedValue>546 describeLoadedValue(const MachineInstr &MI, Register Reg) const override;547 548 unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;549 550 bool isExtendLikelyToBeFolded(MachineInstr &ExtMI,551 MachineRegisterInfo &MRI) const override;552 553 static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset,554 int64_t &NumBytes,555 int64_t &NumPredicateVectors,556 int64_t &NumDataVectors);557 static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset,558 int64_t &ByteSized,559 int64_t &VGSized);560 561 // Return true if address of the form BaseReg + Scale * ScaledReg + Offset can562 // be used for a load/store of NumBytes. BaseReg is always present and563 // implicit.564 bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset,565 unsigned Scale) const;566 567 // Decrement the SP, issuing probes along the way. `TargetReg` is the new top568 // of the stack. `FrameSetup` is passed as true, if the allocation is a part569 // of constructing the activation frame of a function.570 MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI,571 Register TargetReg,572 bool FrameSetup) const;573 574#define GET_INSTRINFO_HELPER_DECLS575#include "AArch64GenInstrInfo.inc"576 577protected:578 /// If the specific machine instruction is an instruction that moves/copies579 /// value from one register to another register return destination and source580 /// registers as machine operands.581 std::optional<DestSourcePair>582 isCopyInstrImpl(const MachineInstr &MI) const override;583 std::optional<DestSourcePair>584 isCopyLikeInstrImpl(const MachineInstr &MI) const override;585 586private:587 unsigned getInstBundleLength(const MachineInstr &MI) const;588 589 /// Sets the offsets on outlined instructions in \p MBB which use SP590 /// so that they will be valid post-outlining.591 ///592 /// \param MBB A \p MachineBasicBlock in an outlined function.593 void fixupPostOutline(MachineBasicBlock &MBB) const;594 595 void instantiateCondBranch(MachineBasicBlock &MBB, const DebugLoc &DL,596 MachineBasicBlock *TBB,597 ArrayRef<MachineOperand> Cond) const;598 bool substituteCmpToZero(MachineInstr &CmpInstr, unsigned SrcReg,599 const MachineRegisterInfo &MRI) const;600 bool removeCmpToZeroOrOne(MachineInstr &CmpInstr, unsigned SrcReg,601 int CmpValue, const MachineRegisterInfo &MRI) const;602 603 /// Returns an unused general-purpose register which can be used for604 /// constructing an outlined call if one exists. Returns 0 otherwise.605 Register findRegisterToSaveLRTo(outliner::Candidate &C) const;606 607 /// Remove a ptest of a predicate-generating operation that already sets, or608 /// can be made to set, the condition codes in an identical manner609 bool optimizePTestInstr(MachineInstr *PTest, unsigned MaskReg,610 unsigned PredReg,611 const MachineRegisterInfo *MRI) const;612 std::optional<unsigned>613 canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,614 MachineInstr *Pred, const MachineRegisterInfo *MRI) const;615 616 /// verifyInstruction - Perform target specific instruction verification.617 bool verifyInstruction(const MachineInstr &MI,618 StringRef &ErrInfo) const override;619};620 621struct UsedNZCV {622 bool N = false;623 bool Z = false;624 bool C = false;625 bool V = false;626 627 UsedNZCV() = default;628 629 UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {630 this->N |= UsedFlags.N;631 this->Z |= UsedFlags.Z;632 this->C |= UsedFlags.C;633 this->V |= UsedFlags.V;634 return *this;635 }636};637 638/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV639/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.640/// \returns std::nullopt otherwise.641///642/// Collect instructions using that flags in \p CCUseInstrs if provided.643std::optional<UsedNZCV>644examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,645 const TargetRegisterInfo &TRI,646 SmallVectorImpl<MachineInstr *> *CCUseInstrs = nullptr);647 648/// Return true if there is an instruction /after/ \p DefMI and before \p UseMI649/// which either reads or clobbers NZCV.650bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,651 const MachineInstr &UseMI,652 const TargetRegisterInfo *TRI);653 654MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg,655 unsigned Reg, const StackOffset &Offset,656 bool LastAdjustmentWasScalable = true);657MCCFIInstruction658createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,659 const StackOffset &OffsetFromDefCFA,660 std::optional<int64_t> IncomingVGOffsetFromDefCFA);661 662/// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg663/// plus Offset. This is intended to be used from within the prolog/epilog664/// insertion (PEI) pass, where a virtual scratch register may be allocated665/// if necessary, to be replaced by the scavenger at the end of PEI.666void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,667 const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,668 StackOffset Offset, const TargetInstrInfo *TII,669 MachineInstr::MIFlag = MachineInstr::NoFlags,670 bool SetNZCV = false, bool NeedsWinCFI = false,671 bool *HasWinCFI = nullptr, bool EmitCFAOffset = false,672 StackOffset InitialOffset = {},673 unsigned FrameReg = AArch64::SP);674 675/// rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the676/// FP. Return false if the offset could not be handled directly in MI, and677/// return the left-over portion by reference.678bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,679 unsigned FrameReg, StackOffset &Offset,680 const AArch64InstrInfo *TII);681 682/// Use to report the frame offset status in isAArch64FrameOffsetLegal.683enum AArch64FrameOffsetStatus {684 AArch64FrameOffsetCannotUpdate = 0x0, ///< Offset cannot apply.685 AArch64FrameOffsetIsLegal = 0x1, ///< Offset is legal.686 AArch64FrameOffsetCanUpdate = 0x2 ///< Offset can apply, at least partly.687};688 689/// Check if the @p Offset is a valid frame offset for @p MI.690/// The returned value reports the validity of the frame offset for @p MI.691/// It uses the values defined by AArch64FrameOffsetStatus for that.692/// If result == AArch64FrameOffsetCannotUpdate, @p MI cannot be updated to693/// use an offset.eq694/// If result & AArch64FrameOffsetIsLegal, @p Offset can completely be695/// rewritten in @p MI.696/// If result & AArch64FrameOffsetCanUpdate, @p Offset contains the697/// amount that is off the limit of the legal offset.698/// If set, @p OutUseUnscaledOp will contain the whether @p MI should be699/// turned into an unscaled operator, which opcode is in @p OutUnscaledOp.700/// If set, @p EmittableOffset contains the amount that can be set in @p MI701/// (possibly with @p OutUnscaledOp if OutUseUnscaledOp is true) and that702/// is a legal offset.703int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset,704 bool *OutUseUnscaledOp = nullptr,705 unsigned *OutUnscaledOp = nullptr,706 int64_t *EmittableOffset = nullptr);707 708static inline bool isUncondBranchOpcode(int Opc) { return Opc == AArch64::B; }709 710static inline bool isCondBranchOpcode(int Opc) {711 switch (Opc) {712 case AArch64::Bcc:713 case AArch64::CBZW:714 case AArch64::CBZX:715 case AArch64::CBNZW:716 case AArch64::CBNZX:717 case AArch64::TBZW:718 case AArch64::TBZX:719 case AArch64::TBNZW:720 case AArch64::TBNZX:721 case AArch64::CBWPri:722 case AArch64::CBXPri:723 case AArch64::CBBAssertExt:724 case AArch64::CBHAssertExt:725 case AArch64::CBWPrr:726 case AArch64::CBXPrr:727 return true;728 default:729 return false;730 }731}732 733static inline bool isIndirectBranchOpcode(int Opc) {734 switch (Opc) {735 case AArch64::BR:736 case AArch64::BRAA:737 case AArch64::BRAB:738 case AArch64::BRAAZ:739 case AArch64::BRABZ:740 return true;741 }742 return false;743}744 745static inline bool isIndirectCallOpcode(unsigned Opc) {746 switch (Opc) {747 case AArch64::BLR:748 case AArch64::BLRAA:749 case AArch64::BLRAB:750 case AArch64::BLRAAZ:751 case AArch64::BLRABZ:752 return true;753 default:754 return false;755 }756}757 758static inline bool isPTrueOpcode(unsigned Opc) {759 switch (Opc) {760 case AArch64::PTRUE_B:761 case AArch64::PTRUE_H:762 case AArch64::PTRUE_S:763 case AArch64::PTRUE_D:764 return true;765 default:766 return false;767 }768}769 770/// Return opcode to be used for indirect calls.771unsigned getBLRCallOpcode(const MachineFunction &MF);772 773/// Return XPAC opcode to be used for a ptrauth strip using the given key.774static inline unsigned getXPACOpcodeForKey(AArch64PACKey::ID K) {775 using namespace AArch64PACKey;776 switch (K) {777 case IA: case IB: return AArch64::XPACI;778 case DA: case DB: return AArch64::XPACD;779 }780 llvm_unreachable("Unhandled AArch64PACKey::ID enum");781}782 783/// Return AUT opcode to be used for a ptrauth auth using the given key, or its784/// AUT*Z variant that doesn't take a discriminator operand, using zero instead.785static inline unsigned getAUTOpcodeForKey(AArch64PACKey::ID K, bool Zero) {786 using namespace AArch64PACKey;787 switch (K) {788 case IA: return Zero ? AArch64::AUTIZA : AArch64::AUTIA;789 case IB: return Zero ? AArch64::AUTIZB : AArch64::AUTIB;790 case DA: return Zero ? AArch64::AUTDZA : AArch64::AUTDA;791 case DB: return Zero ? AArch64::AUTDZB : AArch64::AUTDB;792 }793 llvm_unreachable("Unhandled AArch64PACKey::ID enum");794}795 796/// Return PAC opcode to be used for a ptrauth sign using the given key, or its797/// PAC*Z variant that doesn't take a discriminator operand, using zero instead.798static inline unsigned getPACOpcodeForKey(AArch64PACKey::ID K, bool Zero) {799 using namespace AArch64PACKey;800 switch (K) {801 case IA: return Zero ? AArch64::PACIZA : AArch64::PACIA;802 case IB: return Zero ? AArch64::PACIZB : AArch64::PACIB;803 case DA: return Zero ? AArch64::PACDZA : AArch64::PACDA;804 case DB: return Zero ? AArch64::PACDZB : AArch64::PACDB;805 }806 llvm_unreachable("Unhandled AArch64PACKey::ID enum");807}808 809// struct TSFlags {810#define TSFLAG_ELEMENT_SIZE_TYPE(X) (X) // 3-bits811#define TSFLAG_DESTRUCTIVE_INST_TYPE(X) ((X) << 3) // 4-bits812#define TSFLAG_FALSE_LANE_TYPE(X) ((X) << 7) // 2-bits813#define TSFLAG_INSTR_FLAGS(X) ((X) << 9) // 2-bits814#define TSFLAG_SME_MATRIX_TYPE(X) ((X) << 11) // 3-bits815// }816 817namespace AArch64 {818 819enum ElementSizeType {820 ElementSizeMask = TSFLAG_ELEMENT_SIZE_TYPE(0x7),821 ElementSizeNone = TSFLAG_ELEMENT_SIZE_TYPE(0x0),822 ElementSizeB = TSFLAG_ELEMENT_SIZE_TYPE(0x1),823 ElementSizeH = TSFLAG_ELEMENT_SIZE_TYPE(0x2),824 ElementSizeS = TSFLAG_ELEMENT_SIZE_TYPE(0x3),825 ElementSizeD = TSFLAG_ELEMENT_SIZE_TYPE(0x4),826};827 828enum DestructiveInstType {829 DestructiveInstTypeMask = TSFLAG_DESTRUCTIVE_INST_TYPE(0xf),830 NotDestructive = TSFLAG_DESTRUCTIVE_INST_TYPE(0x0),831 DestructiveOther = TSFLAG_DESTRUCTIVE_INST_TYPE(0x1),832 DestructiveUnary = TSFLAG_DESTRUCTIVE_INST_TYPE(0x2),833 DestructiveBinaryImm = TSFLAG_DESTRUCTIVE_INST_TYPE(0x3),834 DestructiveBinaryShImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0x4),835 DestructiveBinary = TSFLAG_DESTRUCTIVE_INST_TYPE(0x5),836 DestructiveBinaryComm = TSFLAG_DESTRUCTIVE_INST_TYPE(0x6),837 DestructiveBinaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x7),838 DestructiveTernaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x8),839 Destructive2xRegImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0x9),840 DestructiveUnaryPassthru = TSFLAG_DESTRUCTIVE_INST_TYPE(0xa),841};842 843enum FalseLaneType {844 FalseLanesMask = TSFLAG_FALSE_LANE_TYPE(0x3),845 FalseLanesZero = TSFLAG_FALSE_LANE_TYPE(0x1),846 FalseLanesUndef = TSFLAG_FALSE_LANE_TYPE(0x2),847};848 849// NOTE: This is a bit field.850static const uint64_t InstrFlagIsWhile = TSFLAG_INSTR_FLAGS(0x1);851static const uint64_t InstrFlagIsPTestLike = TSFLAG_INSTR_FLAGS(0x2);852 853enum SMEMatrixType {854 SMEMatrixTypeMask = TSFLAG_SME_MATRIX_TYPE(0x7),855 SMEMatrixNone = TSFLAG_SME_MATRIX_TYPE(0x0),856 SMEMatrixTileB = TSFLAG_SME_MATRIX_TYPE(0x1),857 SMEMatrixTileH = TSFLAG_SME_MATRIX_TYPE(0x2),858 SMEMatrixTileS = TSFLAG_SME_MATRIX_TYPE(0x3),859 SMEMatrixTileD = TSFLAG_SME_MATRIX_TYPE(0x4),860 SMEMatrixTileQ = TSFLAG_SME_MATRIX_TYPE(0x5),861 SMEMatrixArray = TSFLAG_SME_MATRIX_TYPE(0x6),862};863 864#undef TSFLAG_ELEMENT_SIZE_TYPE865#undef TSFLAG_DESTRUCTIVE_INST_TYPE866#undef TSFLAG_FALSE_LANE_TYPE867#undef TSFLAG_INSTR_FLAGS868#undef TSFLAG_SME_MATRIX_TYPE869 870int getSVEPseudoMap(uint16_t Opcode);871int getSVERevInstr(uint16_t Opcode);872int getSVENonRevInstr(uint16_t Opcode);873 874int getSMEPseudoMap(uint16_t Opcode);875}876 877} // end namespace llvm878 879#endif880