3324 lines · cpp
1//===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains a pass that expands pseudo instructions into target10// instructions to allow proper scheduling, if-conversion, and other late11// optimizations. This pass should be run after register allocation but before12// the post-regalloc scheduling pass.13//14//===----------------------------------------------------------------------===//15 16#include "ARM.h"17#include "ARMBaseInstrInfo.h"18#include "ARMBaseRegisterInfo.h"19#include "ARMConstantPoolValue.h"20#include "ARMMachineFunctionInfo.h"21#include "ARMSubtarget.h"22#include "MCTargetDesc/ARMAddressingModes.h"23#include "llvm/CodeGen/LivePhysRegs.h"24#include "llvm/CodeGen/MachineFrameInfo.h"25#include "llvm/CodeGen/MachineFunctionPass.h"26#include "llvm/CodeGen/MachineJumpTableInfo.h"27#include "llvm/MC/MCAsmInfo.h"28#include "llvm/Support/Debug.h"29 30#include <atomic>31 32using namespace llvm;33 34#define DEBUG_TYPE "arm-pseudo"35 36static cl::opt<bool>37VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,38 cl::desc("Verify machine code after expanding ARM pseudos"));39 40#define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"41 42namespace {43 class ARMExpandPseudo : public MachineFunctionPass {44 public:45 static char ID;46 ARMExpandPseudo() : MachineFunctionPass(ID) {}47 48 const ARMBaseInstrInfo *TII;49 const TargetRegisterInfo *TRI;50 const ARMSubtarget *STI;51 ARMFunctionInfo *AFI;52 53 bool runOnMachineFunction(MachineFunction &Fn) override;54 55 MachineFunctionProperties getRequiredProperties() const override {56 return MachineFunctionProperties().setNoVRegs();57 }58 59 StringRef getPassName() const override {60 return ARM_EXPAND_PSEUDO_NAME;61 }62 63 private:64 bool ExpandMI(MachineBasicBlock &MBB,65 MachineBasicBlock::iterator MBBI,66 MachineBasicBlock::iterator &NextMBBI);67 bool ExpandMBB(MachineBasicBlock &MBB);68 void ExpandVLD(MachineBasicBlock::iterator &MBBI);69 void ExpandVST(MachineBasicBlock::iterator &MBBI);70 void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);71 void ExpandVTBL(MachineBasicBlock::iterator &MBBI,72 unsigned Opc, bool IsExt);73 void ExpandMQQPRLoadStore(MachineBasicBlock::iterator &MBBI);74 void ExpandTMOV32BitImm(MachineBasicBlock &MBB,75 MachineBasicBlock::iterator &MBBI);76 void ExpandMOV32BitImm(MachineBasicBlock &MBB,77 MachineBasicBlock::iterator &MBBI);78 void CMSEClearGPRegs(MachineBasicBlock &MBB,79 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,80 const SmallVectorImpl<unsigned> &ClearRegs,81 unsigned ClobberReg);82 MachineBasicBlock &CMSEClearFPRegs(MachineBasicBlock &MBB,83 MachineBasicBlock::iterator MBBI);84 MachineBasicBlock &CMSEClearFPRegsV8(MachineBasicBlock &MBB,85 MachineBasicBlock::iterator MBBI,86 const BitVector &ClearRegs);87 MachineBasicBlock &CMSEClearFPRegsV81(MachineBasicBlock &MBB,88 MachineBasicBlock::iterator MBBI,89 const BitVector &ClearRegs);90 void CMSESaveClearFPRegs(MachineBasicBlock &MBB,91 MachineBasicBlock::iterator MBBI, DebugLoc &DL,92 const LivePhysRegs &LiveRegs,93 SmallVectorImpl<unsigned> &AvailableRegs);94 void CMSESaveClearFPRegsV8(MachineBasicBlock &MBB,95 MachineBasicBlock::iterator MBBI, DebugLoc &DL,96 const LivePhysRegs &LiveRegs,97 SmallVectorImpl<unsigned> &ScratchRegs);98 void CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,99 MachineBasicBlock::iterator MBBI, DebugLoc &DL,100 const LivePhysRegs &LiveRegs);101 void CMSERestoreFPRegs(MachineBasicBlock &MBB,102 MachineBasicBlock::iterator MBBI, DebugLoc &DL,103 SmallVectorImpl<unsigned> &AvailableRegs);104 void CMSERestoreFPRegsV8(MachineBasicBlock &MBB,105 MachineBasicBlock::iterator MBBI, DebugLoc &DL,106 SmallVectorImpl<unsigned> &AvailableRegs);107 void CMSERestoreFPRegsV81(MachineBasicBlock &MBB,108 MachineBasicBlock::iterator MBBI, DebugLoc &DL,109 SmallVectorImpl<unsigned> &AvailableRegs);110 bool ExpandCMP_SWAP(MachineBasicBlock &MBB,111 MachineBasicBlock::iterator MBBI, unsigned LdrexOp,112 unsigned StrexOp, unsigned UxtOp,113 MachineBasicBlock::iterator &NextMBBI);114 115 bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,116 MachineBasicBlock::iterator MBBI,117 MachineBasicBlock::iterator &NextMBBI);118 };119 char ARMExpandPseudo::ID = 0;120}121 122INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,123 false)124 125namespace {126 // Constants for register spacing in NEON load/store instructions.127 // For quad-register load-lane and store-lane pseudo instructors, the128 // spacing is initially assumed to be EvenDblSpc, and that is changed to129 // OddDblSpc depending on the lane number operand.130 enum NEONRegSpacing {131 SingleSpc,132 SingleLowSpc , // Single spacing, low registers, three and four vectors.133 SingleHighQSpc, // Single spacing, high registers, four vectors.134 SingleHighTSpc, // Single spacing, high registers, three vectors.135 EvenDblSpc,136 OddDblSpc137 };138 139 // Entries for NEON load/store information table. The table is sorted by140 // PseudoOpc for fast binary-search lookups.141 struct NEONLdStTableEntry {142 uint16_t PseudoOpc;143 uint16_t RealOpc;144 bool IsLoad;145 bool isUpdating;146 bool hasWritebackOperand;147 uint8_t RegSpacing; // One of type NEONRegSpacing148 uint8_t NumRegs; // D registers loaded or stored149 uint8_t RegElts; // elements per D register; used for lane ops150 // FIXME: Temporary flag to denote whether the real instruction takes151 // a single register (like the encoding) or all of the registers in152 // the list (like the asm syntax and the isel DAG). When all definitions153 // are converted to take only the single encoded register, this will154 // go away.155 bool copyAllListRegs;156 157 // Comparison methods for binary search of the table.158 bool operator<(const NEONLdStTableEntry &TE) const {159 return PseudoOpc < TE.PseudoOpc;160 }161 friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {162 return TE.PseudoOpc < PseudoOpc;163 }164 [[maybe_unused]] friend bool operator<(unsigned PseudoOpc,165 const NEONLdStTableEntry &TE) {166 return PseudoOpc < TE.PseudoOpc;167 }168 };169}170 171static const NEONLdStTableEntry NEONLdStTable[] = {172{ ARM::VLD1LNq16Pseudo, ARM::VLD1LNd16, true, false, false, EvenDblSpc, 1, 4 ,true},173{ ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true, EvenDblSpc, 1, 4 ,true},174{ ARM::VLD1LNq32Pseudo, ARM::VLD1LNd32, true, false, false, EvenDblSpc, 1, 2 ,true},175{ ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true, EvenDblSpc, 1, 2 ,true},176{ ARM::VLD1LNq8Pseudo, ARM::VLD1LNd8, true, false, false, EvenDblSpc, 1, 8 ,true},177{ ARM::VLD1LNq8Pseudo_UPD, ARM::VLD1LNd8_UPD, true, true, true, EvenDblSpc, 1, 8 ,true},178 179{ ARM::VLD1d16QPseudo, ARM::VLD1d16Q, true, false, false, SingleSpc, 4, 4 ,false},180{ ARM::VLD1d16QPseudoWB_fixed, ARM::VLD1d16Qwb_fixed, true, true, false, SingleSpc, 4, 4 ,false},181{ ARM::VLD1d16QPseudoWB_register, ARM::VLD1d16Qwb_register, true, true, true, SingleSpc, 4, 4 ,false},182{ ARM::VLD1d16TPseudo, ARM::VLD1d16T, true, false, false, SingleSpc, 3, 4 ,false},183{ ARM::VLD1d16TPseudoWB_fixed, ARM::VLD1d16Twb_fixed, true, true, false, SingleSpc, 3, 4 ,false},184{ ARM::VLD1d16TPseudoWB_register, ARM::VLD1d16Twb_register, true, true, true, SingleSpc, 3, 4 ,false},185 186{ ARM::VLD1d32QPseudo, ARM::VLD1d32Q, true, false, false, SingleSpc, 4, 2 ,false},187{ ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d32Qwb_fixed, true, true, false, SingleSpc, 4, 2 ,false},188{ ARM::VLD1d32QPseudoWB_register, ARM::VLD1d32Qwb_register, true, true, true, SingleSpc, 4, 2 ,false},189{ ARM::VLD1d32TPseudo, ARM::VLD1d32T, true, false, false, SingleSpc, 3, 2 ,false},190{ ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d32Twb_fixed, true, true, false, SingleSpc, 3, 2 ,false},191{ ARM::VLD1d32TPseudoWB_register, ARM::VLD1d32Twb_register, true, true, true, SingleSpc, 3, 2 ,false},192 193{ ARM::VLD1d64QPseudo, ARM::VLD1d64Q, true, false, false, SingleSpc, 4, 1 ,false},194{ ARM::VLD1d64QPseudoWB_fixed, ARM::VLD1d64Qwb_fixed, true, true, false, SingleSpc, 4, 1 ,false},195{ ARM::VLD1d64QPseudoWB_register, ARM::VLD1d64Qwb_register, true, true, true, SingleSpc, 4, 1 ,false},196{ ARM::VLD1d64TPseudo, ARM::VLD1d64T, true, false, false, SingleSpc, 3, 1 ,false},197{ ARM::VLD1d64TPseudoWB_fixed, ARM::VLD1d64Twb_fixed, true, true, false, SingleSpc, 3, 1 ,false},198{ ARM::VLD1d64TPseudoWB_register, ARM::VLD1d64Twb_register, true, true, true, SingleSpc, 3, 1 ,false},199 200{ ARM::VLD1d8QPseudo, ARM::VLD1d8Q, true, false, false, SingleSpc, 4, 8 ,false},201{ ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d8Qwb_fixed, true, true, false, SingleSpc, 4, 8 ,false},202{ ARM::VLD1d8QPseudoWB_register, ARM::VLD1d8Qwb_register, true, true, true, SingleSpc, 4, 8 ,false},203{ ARM::VLD1d8TPseudo, ARM::VLD1d8T, true, false, false, SingleSpc, 3, 8 ,false},204{ ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d8Twb_fixed, true, true, false, SingleSpc, 3, 8 ,false},205{ ARM::VLD1d8TPseudoWB_register, ARM::VLD1d8Twb_register, true, true, true, SingleSpc, 3, 8 ,false},206 207{ ARM::VLD1q16HighQPseudo, ARM::VLD1d16Q, true, false, false, SingleHighQSpc, 4, 4 ,false},208{ ARM::VLD1q16HighQPseudo_UPD, ARM::VLD1d16Qwb_fixed, true, true, true, SingleHighQSpc, 4, 4 ,false},209{ ARM::VLD1q16HighTPseudo, ARM::VLD1d16T, true, false, false, SingleHighTSpc, 3, 4 ,false},210{ ARM::VLD1q16HighTPseudo_UPD, ARM::VLD1d16Twb_fixed, true, true, true, SingleHighTSpc, 3, 4 ,false},211{ ARM::VLD1q16LowQPseudo_UPD, ARM::VLD1d16Qwb_fixed, true, true, true, SingleLowSpc, 4, 4 ,false},212{ ARM::VLD1q16LowTPseudo_UPD, ARM::VLD1d16Twb_fixed, true, true, true, SingleLowSpc, 3, 4 ,false},213 214{ ARM::VLD1q32HighQPseudo, ARM::VLD1d32Q, true, false, false, SingleHighQSpc, 4, 2 ,false},215{ ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1d32Qwb_fixed, true, true, true, SingleHighQSpc, 4, 2 ,false},216{ ARM::VLD1q32HighTPseudo, ARM::VLD1d32T, true, false, false, SingleHighTSpc, 3, 2 ,false},217{ ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1d32Twb_fixed, true, true, true, SingleHighTSpc, 3, 2 ,false},218{ ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1d32Qwb_fixed, true, true, true, SingleLowSpc, 4, 2 ,false},219{ ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1d32Twb_fixed, true, true, true, SingleLowSpc, 3, 2 ,false},220 221{ ARM::VLD1q64HighQPseudo, ARM::VLD1d64Q, true, false, false, SingleHighQSpc, 4, 1 ,false},222{ ARM::VLD1q64HighQPseudo_UPD, ARM::VLD1d64Qwb_fixed, true, true, true, SingleHighQSpc, 4, 1 ,false},223{ ARM::VLD1q64HighTPseudo, ARM::VLD1d64T, true, false, false, SingleHighTSpc, 3, 1 ,false},224{ ARM::VLD1q64HighTPseudo_UPD, ARM::VLD1d64Twb_fixed, true, true, true, SingleHighTSpc, 3, 1 ,false},225{ ARM::VLD1q64LowQPseudo_UPD, ARM::VLD1d64Qwb_fixed, true, true, true, SingleLowSpc, 4, 1 ,false},226{ ARM::VLD1q64LowTPseudo_UPD, ARM::VLD1d64Twb_fixed, true, true, true, SingleLowSpc, 3, 1 ,false},227 228{ ARM::VLD1q8HighQPseudo, ARM::VLD1d8Q, true, false, false, SingleHighQSpc, 4, 8 ,false},229{ ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1d8Qwb_fixed, true, true, true, SingleHighQSpc, 4, 8 ,false},230{ ARM::VLD1q8HighTPseudo, ARM::VLD1d8T, true, false, false, SingleHighTSpc, 3, 8 ,false},231{ ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1d8Twb_fixed, true, true, true, SingleHighTSpc, 3, 8 ,false},232{ ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1d8Qwb_fixed, true, true, true, SingleLowSpc, 4, 8 ,false},233{ ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1d8Twb_fixed, true, true, true, SingleLowSpc, 3, 8 ,false},234 235{ ARM::VLD2DUPq16EvenPseudo, ARM::VLD2DUPd16x2, true, false, false, EvenDblSpc, 2, 4 ,false},236{ ARM::VLD2DUPq16OddPseudo, ARM::VLD2DUPd16x2, true, false, false, OddDblSpc, 2, 4 ,false},237{ ARM::VLD2DUPq16OddPseudoWB_fixed, ARM::VLD2DUPd16x2wb_fixed, true, true, false, OddDblSpc, 2, 4 ,false},238{ ARM::VLD2DUPq16OddPseudoWB_register, ARM::VLD2DUPd16x2wb_register, true, true, true, OddDblSpc, 2, 4 ,false},239{ ARM::VLD2DUPq32EvenPseudo, ARM::VLD2DUPd32x2, true, false, false, EvenDblSpc, 2, 2 ,false},240{ ARM::VLD2DUPq32OddPseudo, ARM::VLD2DUPd32x2, true, false, false, OddDblSpc, 2, 2 ,false},241{ ARM::VLD2DUPq32OddPseudoWB_fixed, ARM::VLD2DUPd32x2wb_fixed, true, true, false, OddDblSpc, 2, 2 ,false},242{ ARM::VLD2DUPq32OddPseudoWB_register, ARM::VLD2DUPd32x2wb_register, true, true, true, OddDblSpc, 2, 2 ,false},243{ ARM::VLD2DUPq8EvenPseudo, ARM::VLD2DUPd8x2, true, false, false, EvenDblSpc, 2, 8 ,false},244{ ARM::VLD2DUPq8OddPseudo, ARM::VLD2DUPd8x2, true, false, false, OddDblSpc, 2, 8 ,false},245{ ARM::VLD2DUPq8OddPseudoWB_fixed, ARM::VLD2DUPd8x2wb_fixed, true, true, false, OddDblSpc, 2, 8 ,false},246{ ARM::VLD2DUPq8OddPseudoWB_register, ARM::VLD2DUPd8x2wb_register, true, true, true, OddDblSpc, 2, 8 ,false},247 248{ ARM::VLD2LNd16Pseudo, ARM::VLD2LNd16, true, false, false, SingleSpc, 2, 4 ,true},249{ ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true, SingleSpc, 2, 4 ,true},250{ ARM::VLD2LNd32Pseudo, ARM::VLD2LNd32, true, false, false, SingleSpc, 2, 2 ,true},251{ ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true, SingleSpc, 2, 2 ,true},252{ ARM::VLD2LNd8Pseudo, ARM::VLD2LNd8, true, false, false, SingleSpc, 2, 8 ,true},253{ ARM::VLD2LNd8Pseudo_UPD, ARM::VLD2LNd8_UPD, true, true, true, SingleSpc, 2, 8 ,true},254{ ARM::VLD2LNq16Pseudo, ARM::VLD2LNq16, true, false, false, EvenDblSpc, 2, 4 ,true},255{ ARM::VLD2LNq16Pseudo_UPD, ARM::VLD2LNq16_UPD, true, true, true, EvenDblSpc, 2, 4 ,true},256{ ARM::VLD2LNq32Pseudo, ARM::VLD2LNq32, true, false, false, EvenDblSpc, 2, 2 ,true},257{ ARM::VLD2LNq32Pseudo_UPD, ARM::VLD2LNq32_UPD, true, true, true, EvenDblSpc, 2, 2 ,true},258 259{ ARM::VLD2q16Pseudo, ARM::VLD2q16, true, false, false, SingleSpc, 4, 4 ,false},260{ ARM::VLD2q16PseudoWB_fixed, ARM::VLD2q16wb_fixed, true, true, false, SingleSpc, 4, 4 ,false},261{ ARM::VLD2q16PseudoWB_register, ARM::VLD2q16wb_register, true, true, true, SingleSpc, 4, 4 ,false},262{ ARM::VLD2q32Pseudo, ARM::VLD2q32, true, false, false, SingleSpc, 4, 2 ,false},263{ ARM::VLD2q32PseudoWB_fixed, ARM::VLD2q32wb_fixed, true, true, false, SingleSpc, 4, 2 ,false},264{ ARM::VLD2q32PseudoWB_register, ARM::VLD2q32wb_register, true, true, true, SingleSpc, 4, 2 ,false},265{ ARM::VLD2q8Pseudo, ARM::VLD2q8, true, false, false, SingleSpc, 4, 8 ,false},266{ ARM::VLD2q8PseudoWB_fixed, ARM::VLD2q8wb_fixed, true, true, false, SingleSpc, 4, 8 ,false},267{ ARM::VLD2q8PseudoWB_register, ARM::VLD2q8wb_register, true, true, true, SingleSpc, 4, 8 ,false},268 269{ ARM::VLD3DUPd16Pseudo, ARM::VLD3DUPd16, true, false, false, SingleSpc, 3, 4,true},270{ ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true, SingleSpc, 3, 4,true},271{ ARM::VLD3DUPd32Pseudo, ARM::VLD3DUPd32, true, false, false, SingleSpc, 3, 2,true},272{ ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true, SingleSpc, 3, 2,true},273{ ARM::VLD3DUPd8Pseudo, ARM::VLD3DUPd8, true, false, false, SingleSpc, 3, 8,true},274{ ARM::VLD3DUPd8Pseudo_UPD, ARM::VLD3DUPd8_UPD, true, true, true, SingleSpc, 3, 8,true},275{ ARM::VLD3DUPq16EvenPseudo, ARM::VLD3DUPq16, true, false, false, EvenDblSpc, 3, 4 ,true},276{ ARM::VLD3DUPq16OddPseudo, ARM::VLD3DUPq16, true, false, false, OddDblSpc, 3, 4 ,true},277{ ARM::VLD3DUPq16OddPseudo_UPD, ARM::VLD3DUPq16_UPD, true, true, true, OddDblSpc, 3, 4 ,true},278{ ARM::VLD3DUPq32EvenPseudo, ARM::VLD3DUPq32, true, false, false, EvenDblSpc, 3, 2 ,true},279{ ARM::VLD3DUPq32OddPseudo, ARM::VLD3DUPq32, true, false, false, OddDblSpc, 3, 2 ,true},280{ ARM::VLD3DUPq32OddPseudo_UPD, ARM::VLD3DUPq32_UPD, true, true, true, OddDblSpc, 3, 2 ,true},281{ ARM::VLD3DUPq8EvenPseudo, ARM::VLD3DUPq8, true, false, false, EvenDblSpc, 3, 8 ,true},282{ ARM::VLD3DUPq8OddPseudo, ARM::VLD3DUPq8, true, false, false, OddDblSpc, 3, 8 ,true},283{ ARM::VLD3DUPq8OddPseudo_UPD, ARM::VLD3DUPq8_UPD, true, true, true, OddDblSpc, 3, 8 ,true},284 285{ ARM::VLD3LNd16Pseudo, ARM::VLD3LNd16, true, false, false, SingleSpc, 3, 4 ,true},286{ ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true, SingleSpc, 3, 4 ,true},287{ ARM::VLD3LNd32Pseudo, ARM::VLD3LNd32, true, false, false, SingleSpc, 3, 2 ,true},288{ ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true, SingleSpc, 3, 2 ,true},289{ ARM::VLD3LNd8Pseudo, ARM::VLD3LNd8, true, false, false, SingleSpc, 3, 8 ,true},290{ ARM::VLD3LNd8Pseudo_UPD, ARM::VLD3LNd8_UPD, true, true, true, SingleSpc, 3, 8 ,true},291{ ARM::VLD3LNq16Pseudo, ARM::VLD3LNq16, true, false, false, EvenDblSpc, 3, 4 ,true},292{ ARM::VLD3LNq16Pseudo_UPD, ARM::VLD3LNq16_UPD, true, true, true, EvenDblSpc, 3, 4 ,true},293{ ARM::VLD3LNq32Pseudo, ARM::VLD3LNq32, true, false, false, EvenDblSpc, 3, 2 ,true},294{ ARM::VLD3LNq32Pseudo_UPD, ARM::VLD3LNq32_UPD, true, true, true, EvenDblSpc, 3, 2 ,true},295 296{ ARM::VLD3d16Pseudo, ARM::VLD3d16, true, false, false, SingleSpc, 3, 4 ,true},297{ ARM::VLD3d16Pseudo_UPD, ARM::VLD3d16_UPD, true, true, true, SingleSpc, 3, 4 ,true},298{ ARM::VLD3d32Pseudo, ARM::VLD3d32, true, false, false, SingleSpc, 3, 2 ,true},299{ ARM::VLD3d32Pseudo_UPD, ARM::VLD3d32_UPD, true, true, true, SingleSpc, 3, 2 ,true},300{ ARM::VLD3d8Pseudo, ARM::VLD3d8, true, false, false, SingleSpc, 3, 8 ,true},301{ ARM::VLD3d8Pseudo_UPD, ARM::VLD3d8_UPD, true, true, true, SingleSpc, 3, 8 ,true},302 303{ ARM::VLD3q16Pseudo_UPD, ARM::VLD3q16_UPD, true, true, true, EvenDblSpc, 3, 4 ,true},304{ ARM::VLD3q16oddPseudo, ARM::VLD3q16, true, false, false, OddDblSpc, 3, 4 ,true},305{ ARM::VLD3q16oddPseudo_UPD, ARM::VLD3q16_UPD, true, true, true, OddDblSpc, 3, 4 ,true},306{ ARM::VLD3q32Pseudo_UPD, ARM::VLD3q32_UPD, true, true, true, EvenDblSpc, 3, 2 ,true},307{ ARM::VLD3q32oddPseudo, ARM::VLD3q32, true, false, false, OddDblSpc, 3, 2 ,true},308{ ARM::VLD3q32oddPseudo_UPD, ARM::VLD3q32_UPD, true, true, true, OddDblSpc, 3, 2 ,true},309{ ARM::VLD3q8Pseudo_UPD, ARM::VLD3q8_UPD, true, true, true, EvenDblSpc, 3, 8 ,true},310{ ARM::VLD3q8oddPseudo, ARM::VLD3q8, true, false, false, OddDblSpc, 3, 8 ,true},311{ ARM::VLD3q8oddPseudo_UPD, ARM::VLD3q8_UPD, true, true, true, OddDblSpc, 3, 8 ,true},312 313{ ARM::VLD4DUPd16Pseudo, ARM::VLD4DUPd16, true, false, false, SingleSpc, 4, 4,true},314{ ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true, SingleSpc, 4, 4,true},315{ ARM::VLD4DUPd32Pseudo, ARM::VLD4DUPd32, true, false, false, SingleSpc, 4, 2,true},316{ ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true, SingleSpc, 4, 2,true},317{ ARM::VLD4DUPd8Pseudo, ARM::VLD4DUPd8, true, false, false, SingleSpc, 4, 8,true},318{ ARM::VLD4DUPd8Pseudo_UPD, ARM::VLD4DUPd8_UPD, true, true, true, SingleSpc, 4, 8,true},319{ ARM::VLD4DUPq16EvenPseudo, ARM::VLD4DUPq16, true, false, false, EvenDblSpc, 4, 4 ,true},320{ ARM::VLD4DUPq16OddPseudo, ARM::VLD4DUPq16, true, false, false, OddDblSpc, 4, 4 ,true},321{ ARM::VLD4DUPq16OddPseudo_UPD, ARM::VLD4DUPq16_UPD, true, true, true, OddDblSpc, 4, 4 ,true},322{ ARM::VLD4DUPq32EvenPseudo, ARM::VLD4DUPq32, true, false, false, EvenDblSpc, 4, 2 ,true},323{ ARM::VLD4DUPq32OddPseudo, ARM::VLD4DUPq32, true, false, false, OddDblSpc, 4, 2 ,true},324{ ARM::VLD4DUPq32OddPseudo_UPD, ARM::VLD4DUPq32_UPD, true, true, true, OddDblSpc, 4, 2 ,true},325{ ARM::VLD4DUPq8EvenPseudo, ARM::VLD4DUPq8, true, false, false, EvenDblSpc, 4, 8 ,true},326{ ARM::VLD4DUPq8OddPseudo, ARM::VLD4DUPq8, true, false, false, OddDblSpc, 4, 8 ,true},327{ ARM::VLD4DUPq8OddPseudo_UPD, ARM::VLD4DUPq8_UPD, true, true, true, OddDblSpc, 4, 8 ,true},328 329{ ARM::VLD4LNd16Pseudo, ARM::VLD4LNd16, true, false, false, SingleSpc, 4, 4 ,true},330{ ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true, SingleSpc, 4, 4 ,true},331{ ARM::VLD4LNd32Pseudo, ARM::VLD4LNd32, true, false, false, SingleSpc, 4, 2 ,true},332{ ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true, SingleSpc, 4, 2 ,true},333{ ARM::VLD4LNd8Pseudo, ARM::VLD4LNd8, true, false, false, SingleSpc, 4, 8 ,true},334{ ARM::VLD4LNd8Pseudo_UPD, ARM::VLD4LNd8_UPD, true, true, true, SingleSpc, 4, 8 ,true},335{ ARM::VLD4LNq16Pseudo, ARM::VLD4LNq16, true, false, false, EvenDblSpc, 4, 4 ,true},336{ ARM::VLD4LNq16Pseudo_UPD, ARM::VLD4LNq16_UPD, true, true, true, EvenDblSpc, 4, 4 ,true},337{ ARM::VLD4LNq32Pseudo, ARM::VLD4LNq32, true, false, false, EvenDblSpc, 4, 2 ,true},338{ ARM::VLD4LNq32Pseudo_UPD, ARM::VLD4LNq32_UPD, true, true, true, EvenDblSpc, 4, 2 ,true},339 340{ ARM::VLD4d16Pseudo, ARM::VLD4d16, true, false, false, SingleSpc, 4, 4 ,true},341{ ARM::VLD4d16Pseudo_UPD, ARM::VLD4d16_UPD, true, true, true, SingleSpc, 4, 4 ,true},342{ ARM::VLD4d32Pseudo, ARM::VLD4d32, true, false, false, SingleSpc, 4, 2 ,true},343{ ARM::VLD4d32Pseudo_UPD, ARM::VLD4d32_UPD, true, true, true, SingleSpc, 4, 2 ,true},344{ ARM::VLD4d8Pseudo, ARM::VLD4d8, true, false, false, SingleSpc, 4, 8 ,true},345{ ARM::VLD4d8Pseudo_UPD, ARM::VLD4d8_UPD, true, true, true, SingleSpc, 4, 8 ,true},346 347{ ARM::VLD4q16Pseudo_UPD, ARM::VLD4q16_UPD, true, true, true, EvenDblSpc, 4, 4 ,true},348{ ARM::VLD4q16oddPseudo, ARM::VLD4q16, true, false, false, OddDblSpc, 4, 4 ,true},349{ ARM::VLD4q16oddPseudo_UPD, ARM::VLD4q16_UPD, true, true, true, OddDblSpc, 4, 4 ,true},350{ ARM::VLD4q32Pseudo_UPD, ARM::VLD4q32_UPD, true, true, true, EvenDblSpc, 4, 2 ,true},351{ ARM::VLD4q32oddPseudo, ARM::VLD4q32, true, false, false, OddDblSpc, 4, 2 ,true},352{ ARM::VLD4q32oddPseudo_UPD, ARM::VLD4q32_UPD, true, true, true, OddDblSpc, 4, 2 ,true},353{ ARM::VLD4q8Pseudo_UPD, ARM::VLD4q8_UPD, true, true, true, EvenDblSpc, 4, 8 ,true},354{ ARM::VLD4q8oddPseudo, ARM::VLD4q8, true, false, false, OddDblSpc, 4, 8 ,true},355{ ARM::VLD4q8oddPseudo_UPD, ARM::VLD4q8_UPD, true, true, true, OddDblSpc, 4, 8 ,true},356 357{ ARM::VST1LNq16Pseudo, ARM::VST1LNd16, false, false, false, EvenDblSpc, 1, 4 ,true},358{ ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true, EvenDblSpc, 1, 4 ,true},359{ ARM::VST1LNq32Pseudo, ARM::VST1LNd32, false, false, false, EvenDblSpc, 1, 2 ,true},360{ ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true, EvenDblSpc, 1, 2 ,true},361{ ARM::VST1LNq8Pseudo, ARM::VST1LNd8, false, false, false, EvenDblSpc, 1, 8 ,true},362{ ARM::VST1LNq8Pseudo_UPD, ARM::VST1LNd8_UPD, false, true, true, EvenDblSpc, 1, 8 ,true},363 364{ ARM::VST1d16QPseudo, ARM::VST1d16Q, false, false, false, SingleSpc, 4, 4 ,false},365{ ARM::VST1d16QPseudoWB_fixed, ARM::VST1d16Qwb_fixed, false, true, false, SingleSpc, 4, 4 ,false},366{ ARM::VST1d16QPseudoWB_register, ARM::VST1d16Qwb_register, false, true, true, SingleSpc, 4, 4 ,false},367{ ARM::VST1d16TPseudo, ARM::VST1d16T, false, false, false, SingleSpc, 3, 4 ,false},368{ ARM::VST1d16TPseudoWB_fixed, ARM::VST1d16Twb_fixed, false, true, false, SingleSpc, 3, 4 ,false},369{ ARM::VST1d16TPseudoWB_register, ARM::VST1d16Twb_register, false, true, true, SingleSpc, 3, 4 ,false},370 371{ ARM::VST1d32QPseudo, ARM::VST1d32Q, false, false, false, SingleSpc, 4, 2 ,false},372{ ARM::VST1d32QPseudoWB_fixed, ARM::VST1d32Qwb_fixed, false, true, false, SingleSpc, 4, 2 ,false},373{ ARM::VST1d32QPseudoWB_register, ARM::VST1d32Qwb_register, false, true, true, SingleSpc, 4, 2 ,false},374{ ARM::VST1d32TPseudo, ARM::VST1d32T, false, false, false, SingleSpc, 3, 2 ,false},375{ ARM::VST1d32TPseudoWB_fixed, ARM::VST1d32Twb_fixed, false, true, false, SingleSpc, 3, 2 ,false},376{ ARM::VST1d32TPseudoWB_register, ARM::VST1d32Twb_register, false, true, true, SingleSpc, 3, 2 ,false},377 378{ ARM::VST1d64QPseudo, ARM::VST1d64Q, false, false, false, SingleSpc, 4, 1 ,false},379{ ARM::VST1d64QPseudoWB_fixed, ARM::VST1d64Qwb_fixed, false, true, false, SingleSpc, 4, 1 ,false},380{ ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true, SingleSpc, 4, 1 ,false},381{ ARM::VST1d64TPseudo, ARM::VST1d64T, false, false, false, SingleSpc, 3, 1 ,false},382{ ARM::VST1d64TPseudoWB_fixed, ARM::VST1d64Twb_fixed, false, true, false, SingleSpc, 3, 1 ,false},383{ ARM::VST1d64TPseudoWB_register, ARM::VST1d64Twb_register, false, true, true, SingleSpc, 3, 1 ,false},384 385{ ARM::VST1d8QPseudo, ARM::VST1d8Q, false, false, false, SingleSpc, 4, 8 ,false},386{ ARM::VST1d8QPseudoWB_fixed, ARM::VST1d8Qwb_fixed, false, true, false, SingleSpc, 4, 8 ,false},387{ ARM::VST1d8QPseudoWB_register, ARM::VST1d8Qwb_register, false, true, true, SingleSpc, 4, 8 ,false},388{ ARM::VST1d8TPseudo, ARM::VST1d8T, false, false, false, SingleSpc, 3, 8 ,false},389{ ARM::VST1d8TPseudoWB_fixed, ARM::VST1d8Twb_fixed, false, true, false, SingleSpc, 3, 8 ,false},390{ ARM::VST1d8TPseudoWB_register, ARM::VST1d8Twb_register, false, true, true, SingleSpc, 3, 8 ,false},391 392{ ARM::VST1q16HighQPseudo, ARM::VST1d16Q, false, false, false, SingleHighQSpc, 4, 4 ,false},393{ ARM::VST1q16HighQPseudo_UPD, ARM::VST1d16Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false},394{ ARM::VST1q16HighTPseudo, ARM::VST1d16T, false, false, false, SingleHighTSpc, 3, 4 ,false},395{ ARM::VST1q16HighTPseudo_UPD, ARM::VST1d16Twb_fixed, false, true, true, SingleHighTSpc, 3, 4 ,false},396{ ARM::VST1q16LowQPseudo_UPD, ARM::VST1d16Qwb_fixed, false, true, true, SingleLowSpc, 4, 4 ,false},397{ ARM::VST1q16LowTPseudo_UPD, ARM::VST1d16Twb_fixed, false, true, true, SingleLowSpc, 3, 4 ,false},398 399{ ARM::VST1q32HighQPseudo, ARM::VST1d32Q, false, false, false, SingleHighQSpc, 4, 2 ,false},400{ ARM::VST1q32HighQPseudo_UPD, ARM::VST1d32Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false},401{ ARM::VST1q32HighTPseudo, ARM::VST1d32T, false, false, false, SingleHighTSpc, 3, 2 ,false},402{ ARM::VST1q32HighTPseudo_UPD, ARM::VST1d32Twb_fixed, false, true, true, SingleHighTSpc, 3, 2 ,false},403{ ARM::VST1q32LowQPseudo_UPD, ARM::VST1d32Qwb_fixed, false, true, true, SingleLowSpc, 4, 2 ,false},404{ ARM::VST1q32LowTPseudo_UPD, ARM::VST1d32Twb_fixed, false, true, true, SingleLowSpc, 3, 2 ,false},405 406{ ARM::VST1q64HighQPseudo, ARM::VST1d64Q, false, false, false, SingleHighQSpc, 4, 1 ,false},407{ ARM::VST1q64HighQPseudo_UPD, ARM::VST1d64Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false},408{ ARM::VST1q64HighTPseudo, ARM::VST1d64T, false, false, false, SingleHighTSpc, 3, 1 ,false},409{ ARM::VST1q64HighTPseudo_UPD, ARM::VST1d64Twb_fixed, false, true, true, SingleHighTSpc, 3, 1 ,false},410{ ARM::VST1q64LowQPseudo_UPD, ARM::VST1d64Qwb_fixed, false, true, true, SingleLowSpc, 4, 1 ,false},411{ ARM::VST1q64LowTPseudo_UPD, ARM::VST1d64Twb_fixed, false, true, true, SingleLowSpc, 3, 1 ,false},412 413{ ARM::VST1q8HighQPseudo, ARM::VST1d8Q, false, false, false, SingleHighQSpc, 4, 8 ,false},414{ ARM::VST1q8HighQPseudo_UPD, ARM::VST1d8Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false},415{ ARM::VST1q8HighTPseudo, ARM::VST1d8T, false, false, false, SingleHighTSpc, 3, 8 ,false},416{ ARM::VST1q8HighTPseudo_UPD, ARM::VST1d8Twb_fixed, false, true, true, SingleHighTSpc, 3, 8 ,false},417{ ARM::VST1q8LowQPseudo_UPD, ARM::VST1d8Qwb_fixed, false, true, true, SingleLowSpc, 4, 8 ,false},418{ ARM::VST1q8LowTPseudo_UPD, ARM::VST1d8Twb_fixed, false, true, true, SingleLowSpc, 3, 8 ,false},419 420{ ARM::VST2LNd16Pseudo, ARM::VST2LNd16, false, false, false, SingleSpc, 2, 4 ,true},421{ ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true, SingleSpc, 2, 4 ,true},422{ ARM::VST2LNd32Pseudo, ARM::VST2LNd32, false, false, false, SingleSpc, 2, 2 ,true},423{ ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true, SingleSpc, 2, 2 ,true},424{ ARM::VST2LNd8Pseudo, ARM::VST2LNd8, false, false, false, SingleSpc, 2, 8 ,true},425{ ARM::VST2LNd8Pseudo_UPD, ARM::VST2LNd8_UPD, false, true, true, SingleSpc, 2, 8 ,true},426{ ARM::VST2LNq16Pseudo, ARM::VST2LNq16, false, false, false, EvenDblSpc, 2, 4,true},427{ ARM::VST2LNq16Pseudo_UPD, ARM::VST2LNq16_UPD, false, true, true, EvenDblSpc, 2, 4,true},428{ ARM::VST2LNq32Pseudo, ARM::VST2LNq32, false, false, false, EvenDblSpc, 2, 2,true},429{ ARM::VST2LNq32Pseudo_UPD, ARM::VST2LNq32_UPD, false, true, true, EvenDblSpc, 2, 2,true},430 431{ ARM::VST2q16Pseudo, ARM::VST2q16, false, false, false, SingleSpc, 4, 4 ,false},432{ ARM::VST2q16PseudoWB_fixed, ARM::VST2q16wb_fixed, false, true, false, SingleSpc, 4, 4 ,false},433{ ARM::VST2q16PseudoWB_register, ARM::VST2q16wb_register, false, true, true, SingleSpc, 4, 4 ,false},434{ ARM::VST2q32Pseudo, ARM::VST2q32, false, false, false, SingleSpc, 4, 2 ,false},435{ ARM::VST2q32PseudoWB_fixed, ARM::VST2q32wb_fixed, false, true, false, SingleSpc, 4, 2 ,false},436{ ARM::VST2q32PseudoWB_register, ARM::VST2q32wb_register, false, true, true, SingleSpc, 4, 2 ,false},437{ ARM::VST2q8Pseudo, ARM::VST2q8, false, false, false, SingleSpc, 4, 8 ,false},438{ ARM::VST2q8PseudoWB_fixed, ARM::VST2q8wb_fixed, false, true, false, SingleSpc, 4, 8 ,false},439{ ARM::VST2q8PseudoWB_register, ARM::VST2q8wb_register, false, true, true, SingleSpc, 4, 8 ,false},440 441{ ARM::VST3LNd16Pseudo, ARM::VST3LNd16, false, false, false, SingleSpc, 3, 4 ,true},442{ ARM::VST3LNd16Pseudo_UPD, ARM::VST3LNd16_UPD, false, true, true, SingleSpc, 3, 4 ,true},443{ ARM::VST3LNd32Pseudo, ARM::VST3LNd32, false, false, false, SingleSpc, 3, 2 ,true},444{ ARM::VST3LNd32Pseudo_UPD, ARM::VST3LNd32_UPD, false, true, true, SingleSpc, 3, 2 ,true},445{ ARM::VST3LNd8Pseudo, ARM::VST3LNd8, false, false, false, SingleSpc, 3, 8 ,true},446{ ARM::VST3LNd8Pseudo_UPD, ARM::VST3LNd8_UPD, false, true, true, SingleSpc, 3, 8 ,true},447{ ARM::VST3LNq16Pseudo, ARM::VST3LNq16, false, false, false, EvenDblSpc, 3, 4,true},448{ ARM::VST3LNq16Pseudo_UPD, ARM::VST3LNq16_UPD, false, true, true, EvenDblSpc, 3, 4,true},449{ ARM::VST3LNq32Pseudo, ARM::VST3LNq32, false, false, false, EvenDblSpc, 3, 2,true},450{ ARM::VST3LNq32Pseudo_UPD, ARM::VST3LNq32_UPD, false, true, true, EvenDblSpc, 3, 2,true},451 452{ ARM::VST3d16Pseudo, ARM::VST3d16, false, false, false, SingleSpc, 3, 4 ,true},453{ ARM::VST3d16Pseudo_UPD, ARM::VST3d16_UPD, false, true, true, SingleSpc, 3, 4 ,true},454{ ARM::VST3d32Pseudo, ARM::VST3d32, false, false, false, SingleSpc, 3, 2 ,true},455{ ARM::VST3d32Pseudo_UPD, ARM::VST3d32_UPD, false, true, true, SingleSpc, 3, 2 ,true},456{ ARM::VST3d8Pseudo, ARM::VST3d8, false, false, false, SingleSpc, 3, 8 ,true},457{ ARM::VST3d8Pseudo_UPD, ARM::VST3d8_UPD, false, true, true, SingleSpc, 3, 8 ,true},458 459{ ARM::VST3q16Pseudo_UPD, ARM::VST3q16_UPD, false, true, true, EvenDblSpc, 3, 4 ,true},460{ ARM::VST3q16oddPseudo, ARM::VST3q16, false, false, false, OddDblSpc, 3, 4 ,true},461{ ARM::VST3q16oddPseudo_UPD, ARM::VST3q16_UPD, false, true, true, OddDblSpc, 3, 4 ,true},462{ ARM::VST3q32Pseudo_UPD, ARM::VST3q32_UPD, false, true, true, EvenDblSpc, 3, 2 ,true},463{ ARM::VST3q32oddPseudo, ARM::VST3q32, false, false, false, OddDblSpc, 3, 2 ,true},464{ ARM::VST3q32oddPseudo_UPD, ARM::VST3q32_UPD, false, true, true, OddDblSpc, 3, 2 ,true},465{ ARM::VST3q8Pseudo_UPD, ARM::VST3q8_UPD, false, true, true, EvenDblSpc, 3, 8 ,true},466{ ARM::VST3q8oddPseudo, ARM::VST3q8, false, false, false, OddDblSpc, 3, 8 ,true},467{ ARM::VST3q8oddPseudo_UPD, ARM::VST3q8_UPD, false, true, true, OddDblSpc, 3, 8 ,true},468 469{ ARM::VST4LNd16Pseudo, ARM::VST4LNd16, false, false, false, SingleSpc, 4, 4 ,true},470{ ARM::VST4LNd16Pseudo_UPD, ARM::VST4LNd16_UPD, false, true, true, SingleSpc, 4, 4 ,true},471{ ARM::VST4LNd32Pseudo, ARM::VST4LNd32, false, false, false, SingleSpc, 4, 2 ,true},472{ ARM::VST4LNd32Pseudo_UPD, ARM::VST4LNd32_UPD, false, true, true, SingleSpc, 4, 2 ,true},473{ ARM::VST4LNd8Pseudo, ARM::VST4LNd8, false, false, false, SingleSpc, 4, 8 ,true},474{ ARM::VST4LNd8Pseudo_UPD, ARM::VST4LNd8_UPD, false, true, true, SingleSpc, 4, 8 ,true},475{ ARM::VST4LNq16Pseudo, ARM::VST4LNq16, false, false, false, EvenDblSpc, 4, 4,true},476{ ARM::VST4LNq16Pseudo_UPD, ARM::VST4LNq16_UPD, false, true, true, EvenDblSpc, 4, 4,true},477{ ARM::VST4LNq32Pseudo, ARM::VST4LNq32, false, false, false, EvenDblSpc, 4, 2,true},478{ ARM::VST4LNq32Pseudo_UPD, ARM::VST4LNq32_UPD, false, true, true, EvenDblSpc, 4, 2,true},479 480{ ARM::VST4d16Pseudo, ARM::VST4d16, false, false, false, SingleSpc, 4, 4 ,true},481{ ARM::VST4d16Pseudo_UPD, ARM::VST4d16_UPD, false, true, true, SingleSpc, 4, 4 ,true},482{ ARM::VST4d32Pseudo, ARM::VST4d32, false, false, false, SingleSpc, 4, 2 ,true},483{ ARM::VST4d32Pseudo_UPD, ARM::VST4d32_UPD, false, true, true, SingleSpc, 4, 2 ,true},484{ ARM::VST4d8Pseudo, ARM::VST4d8, false, false, false, SingleSpc, 4, 8 ,true},485{ ARM::VST4d8Pseudo_UPD, ARM::VST4d8_UPD, false, true, true, SingleSpc, 4, 8 ,true},486 487{ ARM::VST4q16Pseudo_UPD, ARM::VST4q16_UPD, false, true, true, EvenDblSpc, 4, 4 ,true},488{ ARM::VST4q16oddPseudo, ARM::VST4q16, false, false, false, OddDblSpc, 4, 4 ,true},489{ ARM::VST4q16oddPseudo_UPD, ARM::VST4q16_UPD, false, true, true, OddDblSpc, 4, 4 ,true},490{ ARM::VST4q32Pseudo_UPD, ARM::VST4q32_UPD, false, true, true, EvenDblSpc, 4, 2 ,true},491{ ARM::VST4q32oddPseudo, ARM::VST4q32, false, false, false, OddDblSpc, 4, 2 ,true},492{ ARM::VST4q32oddPseudo_UPD, ARM::VST4q32_UPD, false, true, true, OddDblSpc, 4, 2 ,true},493{ ARM::VST4q8Pseudo_UPD, ARM::VST4q8_UPD, false, true, true, EvenDblSpc, 4, 8 ,true},494{ ARM::VST4q8oddPseudo, ARM::VST4q8, false, false, false, OddDblSpc, 4, 8 ,true},495{ ARM::VST4q8oddPseudo_UPD, ARM::VST4q8_UPD, false, true, true, OddDblSpc, 4, 8 ,true}496};497 498/// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON499/// load or store pseudo instruction.500static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {501#ifndef NDEBUG502 // Make sure the table is sorted.503 static std::atomic<bool> TableChecked(false);504 if (!TableChecked.load(std::memory_order_relaxed)) {505 assert(llvm::is_sorted(NEONLdStTable) && "NEONLdStTable is not sorted!");506 TableChecked.store(true, std::memory_order_relaxed);507 }508#endif509 510 auto I = llvm::lower_bound(NEONLdStTable, Opcode);511 if (I != std::end(NEONLdStTable) && I->PseudoOpc == Opcode)512 return I;513 return nullptr;514}515 516/// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,517/// corresponding to the specified register spacing. Not all of the results518/// are necessarily valid, e.g., a Q register only has 2 D subregisters.519static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,520 const TargetRegisterInfo *TRI, MCRegister &D0,521 MCRegister &D1, MCRegister &D2, MCRegister &D3) {522 if (RegSpc == SingleSpc || RegSpc == SingleLowSpc) {523 D0 = TRI->getSubReg(Reg, ARM::dsub_0);524 D1 = TRI->getSubReg(Reg, ARM::dsub_1);525 D2 = TRI->getSubReg(Reg, ARM::dsub_2);526 D3 = TRI->getSubReg(Reg, ARM::dsub_3);527 } else if (RegSpc == SingleHighQSpc) {528 D0 = TRI->getSubReg(Reg, ARM::dsub_4);529 D1 = TRI->getSubReg(Reg, ARM::dsub_5);530 D2 = TRI->getSubReg(Reg, ARM::dsub_6);531 D3 = TRI->getSubReg(Reg, ARM::dsub_7);532 } else if (RegSpc == SingleHighTSpc) {533 D0 = TRI->getSubReg(Reg, ARM::dsub_3);534 D1 = TRI->getSubReg(Reg, ARM::dsub_4);535 D2 = TRI->getSubReg(Reg, ARM::dsub_5);536 D3 = TRI->getSubReg(Reg, ARM::dsub_6);537 } else if (RegSpc == EvenDblSpc) {538 D0 = TRI->getSubReg(Reg, ARM::dsub_0);539 D1 = TRI->getSubReg(Reg, ARM::dsub_2);540 D2 = TRI->getSubReg(Reg, ARM::dsub_4);541 D3 = TRI->getSubReg(Reg, ARM::dsub_6);542 } else {543 assert(RegSpc == OddDblSpc && "unknown register spacing");544 D0 = TRI->getSubReg(Reg, ARM::dsub_1);545 D1 = TRI->getSubReg(Reg, ARM::dsub_3);546 D2 = TRI->getSubReg(Reg, ARM::dsub_5);547 D3 = TRI->getSubReg(Reg, ARM::dsub_7);548 }549}550 551/// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register552/// operands to real VLD instructions with D register operands.553void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {554 MachineInstr &MI = *MBBI;555 MachineBasicBlock &MBB = *MI.getParent();556 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());557 558 const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());559 assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");560 NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;561 unsigned NumRegs = TableEntry->NumRegs;562 563 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),564 TII->get(TableEntry->RealOpc));565 unsigned OpIdx = 0;566 567 bool DstIsDead = MI.getOperand(OpIdx).isDead();568 Register DstReg = MI.getOperand(OpIdx++).getReg();569 570 bool IsVLD2DUP = TableEntry->RealOpc == ARM::VLD2DUPd8x2 ||571 TableEntry->RealOpc == ARM::VLD2DUPd16x2 ||572 TableEntry->RealOpc == ARM::VLD2DUPd32x2 ||573 TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||574 TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||575 TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed ||576 TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_register ||577 TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_register ||578 TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_register;579 580 if (IsVLD2DUP) {581 unsigned SubRegIndex;582 if (RegSpc == EvenDblSpc) {583 SubRegIndex = ARM::dsub_0;584 } else {585 assert(RegSpc == OddDblSpc && "Unexpected spacing!");586 SubRegIndex = ARM::dsub_1;587 }588 Register SubReg = TRI->getSubReg(DstReg, SubRegIndex);589 MCRegister DstRegPair =590 TRI->getMatchingSuperReg(SubReg, ARM::dsub_0, &ARM::DPairSpcRegClass);591 MIB.addReg(DstRegPair, RegState::Define | getDeadRegState(DstIsDead));592 } else {593 MCRegister D0, D1, D2, D3;594 GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);595 MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));596 if (NumRegs > 1 && TableEntry->copyAllListRegs)597 MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));598 if (NumRegs > 2 && TableEntry->copyAllListRegs)599 MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));600 if (NumRegs > 3 && TableEntry->copyAllListRegs)601 MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));602 }603 604 if (TableEntry->isUpdating)605 MIB.add(MI.getOperand(OpIdx++));606 607 // Copy the addrmode6 operands.608 MIB.add(MI.getOperand(OpIdx++));609 MIB.add(MI.getOperand(OpIdx++));610 611 // Copy the am6offset operand.612 if (TableEntry->hasWritebackOperand) {613 // TODO: The writing-back pseudo instructions we translate here are all614 // defined to take am6offset nodes that are capable to represent both fixed615 // and register forms. Some real instructions, however, do not rely on616 // am6offset and have separate definitions for such forms. When this is the617 // case, fixed forms do not take any offset nodes, so here we skip them for618 // such instructions. Once all real and pseudo writing-back instructions are619 // rewritten without use of am6offset nodes, this code will go away.620 const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);621 if (TableEntry->RealOpc == ARM::VLD1d8Qwb_fixed ||622 TableEntry->RealOpc == ARM::VLD1d16Qwb_fixed ||623 TableEntry->RealOpc == ARM::VLD1d32Qwb_fixed ||624 TableEntry->RealOpc == ARM::VLD1d64Qwb_fixed ||625 TableEntry->RealOpc == ARM::VLD1d8Twb_fixed ||626 TableEntry->RealOpc == ARM::VLD1d16Twb_fixed ||627 TableEntry->RealOpc == ARM::VLD1d32Twb_fixed ||628 TableEntry->RealOpc == ARM::VLD1d64Twb_fixed ||629 TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||630 TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||631 TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed) {632 assert(AM6Offset.getReg() == 0 &&633 "A fixed writing-back pseudo instruction provides an offset "634 "register!");635 } else {636 MIB.add(AM6Offset);637 }638 }639 640 // For an instruction writing double-spaced subregs, the pseudo instruction641 // has an extra operand that is a use of the super-register. Record the642 // operand index and skip over it.643 unsigned SrcOpIdx = 0;644 if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc || RegSpc == SingleLowSpc ||645 RegSpc == SingleHighQSpc || RegSpc == SingleHighTSpc)646 SrcOpIdx = OpIdx++;647 648 // Copy the predicate operands.649 MIB.add(MI.getOperand(OpIdx++));650 MIB.add(MI.getOperand(OpIdx++));651 652 // Copy the super-register source operand used for double-spaced subregs over653 // to the new instruction as an implicit operand.654 if (SrcOpIdx != 0) {655 MachineOperand MO = MI.getOperand(SrcOpIdx);656 MO.setImplicit(true);657 MIB.add(MO);658 }659 // Add an implicit def for the super-register.660 MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));661 MIB.copyImplicitOps(MI);662 663 // Transfer memoperands.664 MIB.cloneMemRefs(MI);665 MI.eraseFromParent();666 LLVM_DEBUG(dbgs() << "To: "; MIB.getInstr()->dump(););667}668 669/// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register670/// operands to real VST instructions with D register operands.671void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {672 MachineInstr &MI = *MBBI;673 MachineBasicBlock &MBB = *MI.getParent();674 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());675 676 const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());677 assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");678 NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;679 unsigned NumRegs = TableEntry->NumRegs;680 681 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),682 TII->get(TableEntry->RealOpc));683 unsigned OpIdx = 0;684 if (TableEntry->isUpdating)685 MIB.add(MI.getOperand(OpIdx++));686 687 // Copy the addrmode6 operands.688 MIB.add(MI.getOperand(OpIdx++));689 MIB.add(MI.getOperand(OpIdx++));690 691 if (TableEntry->hasWritebackOperand) {692 // TODO: The writing-back pseudo instructions we translate here are all693 // defined to take am6offset nodes that are capable to represent both fixed694 // and register forms. Some real instructions, however, do not rely on695 // am6offset and have separate definitions for such forms. When this is the696 // case, fixed forms do not take any offset nodes, so here we skip them for697 // such instructions. Once all real and pseudo writing-back instructions are698 // rewritten without use of am6offset nodes, this code will go away.699 const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);700 if (TableEntry->RealOpc == ARM::VST1d8Qwb_fixed ||701 TableEntry->RealOpc == ARM::VST1d16Qwb_fixed ||702 TableEntry->RealOpc == ARM::VST1d32Qwb_fixed ||703 TableEntry->RealOpc == ARM::VST1d64Qwb_fixed ||704 TableEntry->RealOpc == ARM::VST1d8Twb_fixed ||705 TableEntry->RealOpc == ARM::VST1d16Twb_fixed ||706 TableEntry->RealOpc == ARM::VST1d32Twb_fixed ||707 TableEntry->RealOpc == ARM::VST1d64Twb_fixed) {708 assert(AM6Offset.getReg() == 0 &&709 "A fixed writing-back pseudo instruction provides an offset "710 "register!");711 } else {712 MIB.add(AM6Offset);713 }714 }715 716 bool SrcIsKill = MI.getOperand(OpIdx).isKill();717 bool SrcIsUndef = MI.getOperand(OpIdx).isUndef();718 Register SrcReg = MI.getOperand(OpIdx++).getReg();719 MCRegister D0, D1, D2, D3;720 GetDSubRegs(SrcReg, RegSpc, TRI, D0, D1, D2, D3);721 MIB.addReg(D0, getUndefRegState(SrcIsUndef));722 if (NumRegs > 1 && TableEntry->copyAllListRegs)723 MIB.addReg(D1, getUndefRegState(SrcIsUndef));724 if (NumRegs > 2 && TableEntry->copyAllListRegs)725 MIB.addReg(D2, getUndefRegState(SrcIsUndef));726 if (NumRegs > 3 && TableEntry->copyAllListRegs)727 MIB.addReg(D3, getUndefRegState(SrcIsUndef));728 729 // Copy the predicate operands.730 MIB.add(MI.getOperand(OpIdx++));731 MIB.add(MI.getOperand(OpIdx++));732 733 if (SrcIsKill && !SrcIsUndef) // Add an implicit kill for the super-reg.734 MIB->addRegisterKilled(SrcReg, TRI, true);735 else if (!SrcIsUndef)736 MIB.addReg(SrcReg, RegState::Implicit); // Add implicit uses for src reg.737 MIB.copyImplicitOps(MI);738 739 // Transfer memoperands.740 MIB.cloneMemRefs(MI);741 MI.eraseFromParent();742 LLVM_DEBUG(dbgs() << "To: "; MIB.getInstr()->dump(););743}744 745/// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ746/// register operands to real instructions with D register operands.747void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {748 MachineInstr &MI = *MBBI;749 MachineBasicBlock &MBB = *MI.getParent();750 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());751 752 const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());753 assert(TableEntry && "NEONLdStTable lookup failed");754 NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;755 unsigned NumRegs = TableEntry->NumRegs;756 unsigned RegElts = TableEntry->RegElts;757 758 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),759 TII->get(TableEntry->RealOpc));760 unsigned OpIdx = 0;761 // The lane operand is always the 3rd from last operand, before the 2762 // predicate operands.763 unsigned Lane = MI.getOperand(MI.getDesc().getNumOperands() - 3).getImm();764 765 // Adjust the lane and spacing as needed for Q registers.766 assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");767 if (RegSpc == EvenDblSpc && Lane >= RegElts) {768 RegSpc = OddDblSpc;769 Lane -= RegElts;770 }771 assert(Lane < RegElts && "out of range lane for VLD/VST-lane");772 773 MCRegister D0, D1, D2, D3;774 unsigned DstReg = 0;775 bool DstIsDead = false;776 if (TableEntry->IsLoad) {777 DstIsDead = MI.getOperand(OpIdx).isDead();778 DstReg = MI.getOperand(OpIdx++).getReg();779 GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);780 MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));781 if (NumRegs > 1)782 MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));783 if (NumRegs > 2)784 MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));785 if (NumRegs > 3)786 MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));787 }788 789 if (TableEntry->isUpdating)790 MIB.add(MI.getOperand(OpIdx++));791 792 // Copy the addrmode6 operands.793 MIB.add(MI.getOperand(OpIdx++));794 MIB.add(MI.getOperand(OpIdx++));795 // Copy the am6offset operand.796 if (TableEntry->hasWritebackOperand)797 MIB.add(MI.getOperand(OpIdx++));798 799 // Grab the super-register source.800 MachineOperand MO = MI.getOperand(OpIdx++);801 if (!TableEntry->IsLoad)802 GetDSubRegs(MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);803 804 // Add the subregs as sources of the new instruction.805 unsigned SrcFlags = (getUndefRegState(MO.isUndef()) |806 getKillRegState(MO.isKill()));807 MIB.addReg(D0, SrcFlags);808 if (NumRegs > 1)809 MIB.addReg(D1, SrcFlags);810 if (NumRegs > 2)811 MIB.addReg(D2, SrcFlags);812 if (NumRegs > 3)813 MIB.addReg(D3, SrcFlags);814 815 // Add the lane number operand.816 MIB.addImm(Lane);817 OpIdx += 1;818 819 // Copy the predicate operands.820 MIB.add(MI.getOperand(OpIdx++));821 MIB.add(MI.getOperand(OpIdx++));822 823 // Copy the super-register source to be an implicit source.824 MO.setImplicit(true);825 MIB.add(MO);826 if (TableEntry->IsLoad)827 // Add an implicit def for the super-register.828 MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));829 MIB.copyImplicitOps(MI);830 // Transfer memoperands.831 MIB.cloneMemRefs(MI);832 MI.eraseFromParent();833}834 835/// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ836/// register operands to real instructions with D register operands.837void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,838 unsigned Opc, bool IsExt) {839 MachineInstr &MI = *MBBI;840 MachineBasicBlock &MBB = *MI.getParent();841 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());842 843 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc));844 unsigned OpIdx = 0;845 846 // Transfer the destination register operand.847 MIB.add(MI.getOperand(OpIdx++));848 if (IsExt) {849 MachineOperand VdSrc(MI.getOperand(OpIdx++));850 MIB.add(VdSrc);851 }852 853 bool SrcIsKill = MI.getOperand(OpIdx).isKill();854 Register SrcReg = MI.getOperand(OpIdx++).getReg();855 MCRegister D0, D1, D2, D3;856 GetDSubRegs(SrcReg, SingleSpc, TRI, D0, D1, D2, D3);857 MIB.addReg(D0);858 859 // Copy the other source register operand.860 MachineOperand VmSrc(MI.getOperand(OpIdx++));861 MIB.add(VmSrc);862 863 // Copy the predicate operands.864 MIB.add(MI.getOperand(OpIdx++));865 MIB.add(MI.getOperand(OpIdx++));866 867 // Add an implicit kill and use for the super-reg.868 MIB.addReg(SrcReg, RegState::Implicit | getKillRegState(SrcIsKill));869 MIB.copyImplicitOps(MI);870 MI.eraseFromParent();871 LLVM_DEBUG(dbgs() << "To: "; MIB.getInstr()->dump(););872}873 874void ARMExpandPseudo::ExpandMQQPRLoadStore(MachineBasicBlock::iterator &MBBI) {875 MachineInstr &MI = *MBBI;876 MachineBasicBlock &MBB = *MI.getParent();877 unsigned NewOpc =878 MI.getOpcode() == ARM::MQQPRStore || MI.getOpcode() == ARM::MQQQQPRStore879 ? ARM::VSTMDIA880 : ARM::VLDMDIA;881 MachineInstrBuilder MIB =882 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));883 884 unsigned Flags = getKillRegState(MI.getOperand(0).isKill()) |885 getDefRegState(MI.getOperand(0).isDef());886 Register SrcReg = MI.getOperand(0).getReg();887 888 // Copy the destination register.889 MIB.add(MI.getOperand(1));890 MIB.add(predOps(ARMCC::AL));891 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_0), Flags);892 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_1), Flags);893 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_2), Flags);894 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_3), Flags);895 if (MI.getOpcode() == ARM::MQQQQPRStore ||896 MI.getOpcode() == ARM::MQQQQPRLoad) {897 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_4), Flags);898 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_5), Flags);899 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_6), Flags);900 MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_7), Flags);901 }902 903 if (NewOpc == ARM::VSTMDIA)904 MIB.addReg(SrcReg, RegState::Implicit);905 906 MIB.copyImplicitOps(MI);907 MIB.cloneMemRefs(MI);908 MI.eraseFromParent();909}910 911static bool IsAnAddressOperand(const MachineOperand &MO) {912 // This check is overly conservative. Unless we are certain that the machine913 // operand is not a symbol reference, we return that it is a symbol reference.914 // This is important as the load pair may not be split up Windows.915 switch (MO.getType()) {916 case MachineOperand::MO_Register:917 case MachineOperand::MO_Immediate:918 case MachineOperand::MO_CImmediate:919 case MachineOperand::MO_FPImmediate:920 case MachineOperand::MO_ShuffleMask:921 return false;922 case MachineOperand::MO_MachineBasicBlock:923 return true;924 case MachineOperand::MO_FrameIndex:925 return false;926 case MachineOperand::MO_ConstantPoolIndex:927 case MachineOperand::MO_TargetIndex:928 case MachineOperand::MO_JumpTableIndex:929 case MachineOperand::MO_ExternalSymbol:930 case MachineOperand::MO_GlobalAddress:931 case MachineOperand::MO_BlockAddress:932 return true;933 case MachineOperand::MO_RegisterMask:934 case MachineOperand::MO_RegisterLiveOut:935 return false;936 case MachineOperand::MO_Metadata:937 case MachineOperand::MO_MCSymbol:938 return true;939 case MachineOperand::MO_DbgInstrRef:940 case MachineOperand::MO_CFIIndex:941 return false;942 case MachineOperand::MO_IntrinsicID:943 case MachineOperand::MO_Predicate:944 llvm_unreachable("should not exist post-isel");945 }946 llvm_unreachable("unhandled machine operand type");947}948 949static MachineOperand makeImplicit(const MachineOperand &MO) {950 MachineOperand NewMO = MO;951 NewMO.setImplicit();952 return NewMO;953}954 955static MachineOperand getMovOperand(const MachineOperand &MO,956 unsigned TargetFlag) {957 unsigned TF = MO.getTargetFlags() | TargetFlag;958 switch (MO.getType()) {959 case MachineOperand::MO_Immediate: {960 unsigned Imm = MO.getImm();961 switch (TargetFlag) {962 case ARMII::MO_HI_8_15:963 Imm = (Imm >> 24) & 0xff;964 break;965 case ARMII::MO_HI_0_7:966 Imm = (Imm >> 16) & 0xff;967 break;968 case ARMII::MO_LO_8_15:969 Imm = (Imm >> 8) & 0xff;970 break;971 case ARMII::MO_LO_0_7:972 Imm = Imm & 0xff;973 break;974 case ARMII::MO_HI16:975 Imm = (Imm >> 16) & 0xffff;976 break;977 case ARMII::MO_LO16:978 Imm = Imm & 0xffff;979 break;980 default:981 llvm_unreachable("Only HI/LO target flags are expected");982 }983 return MachineOperand::CreateImm(Imm);984 }985 case MachineOperand::MO_ExternalSymbol:986 return MachineOperand::CreateES(MO.getSymbolName(), TF);987 case MachineOperand::MO_JumpTableIndex:988 return MachineOperand::CreateJTI(MO.getIndex(), TF);989 default:990 return MachineOperand::CreateGA(MO.getGlobal(), MO.getOffset(), TF);991 }992}993 994void ARMExpandPseudo::ExpandTMOV32BitImm(MachineBasicBlock &MBB,995 MachineBasicBlock::iterator &MBBI) {996 MachineInstr &MI = *MBBI;997 Register DstReg = MI.getOperand(0).getReg();998 bool DstIsDead = MI.getOperand(0).isDead();999 const MachineOperand &MO = MI.getOperand(1);1000 unsigned MIFlags = MI.getFlags();1001 1002 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());1003 1004 // Expand the mov into a sequence of mov/add+lsl of the individual bytes. We1005 // want to avoid emitting any zero bytes, as they won't change the result, and1006 // also don't want any pointless shifts, so instead of immediately emitting1007 // the shift for a byte we keep track of how much we will need to shift and do1008 // it before the next nonzero byte.1009 unsigned PendingShift = 0;1010 for (unsigned Byte = 0; Byte < 4; ++Byte) {1011 unsigned Flag = Byte == 0 ? ARMII::MO_HI_8_151012 : Byte == 1 ? ARMII::MO_HI_0_71013 : Byte == 2 ? ARMII::MO_LO_8_151014 : ARMII::MO_LO_0_7;1015 MachineOperand Operand = getMovOperand(MO, Flag);1016 bool ZeroImm = Operand.isImm() && Operand.getImm() == 0;1017 unsigned Op = PendingShift ? ARM::tADDi8 : ARM::tMOVi8;1018 1019 // Emit the pending shift if we're going to emit this byte or if we've1020 // reached the end.1021 if (PendingShift && (!ZeroImm || Byte == 3)) {1022 MachineInstr *Lsl =1023 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tLSLri), DstReg)1024 .add(t1CondCodeOp(true))1025 .addReg(DstReg)1026 .addImm(PendingShift)1027 .add(predOps(ARMCC::AL))1028 .setMIFlags(MIFlags);1029 (void)Lsl;1030 LLVM_DEBUG(dbgs() << "And: "; Lsl->dump(););1031 PendingShift = 0;1032 }1033 1034 // Emit this byte if it's nonzero.1035 if (!ZeroImm) {1036 MachineInstrBuilder MIB =1037 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Op), DstReg)1038 .add(t1CondCodeOp(true));1039 if (Op == ARM::tADDi8)1040 MIB.addReg(DstReg);1041 MIB.add(Operand);1042 MIB.add(predOps(ARMCC::AL));1043 MIB.setMIFlags(MIFlags);1044 LLVM_DEBUG(dbgs() << (Op == ARM::tMOVi8 ? "To: " : "And:") << " ";1045 MIB.getInstr()->dump(););1046 }1047 1048 // Don't accumulate the shift value if we've not yet seen a nonzero byte.1049 if (PendingShift || !ZeroImm)1050 PendingShift += 8;1051 }1052 1053 // The dest is dead on the last instruction we emitted if it was dead on the1054 // original instruction.1055 (--MBBI)->getOperand(0).setIsDead(DstIsDead);1056 1057 MI.eraseFromParent();1058}1059 1060void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,1061 MachineBasicBlock::iterator &MBBI) {1062 MachineInstr &MI = *MBBI;1063 unsigned Opcode = MI.getOpcode();1064 Register PredReg;1065 ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);1066 Register DstReg = MI.getOperand(0).getReg();1067 bool DstIsDead = MI.getOperand(0).isDead();1068 bool isCC = Opcode == ARM::MOVCCi32imm || Opcode == ARM::t2MOVCCi32imm;1069 const MachineOperand &MO = MI.getOperand(isCC ? 2 : 1);1070 bool RequiresBundling = STI->isTargetWindows() && IsAnAddressOperand(MO);1071 MachineInstrBuilder LO16, HI16;1072 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());1073 1074 if (!STI->hasV6T2Ops() &&1075 (Opcode == ARM::MOVi32imm || Opcode == ARM::MOVCCi32imm)) {1076 // FIXME Windows CE supports older ARM CPUs1077 assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");1078 1079 assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");1080 unsigned ImmVal = (unsigned)MO.getImm();1081 unsigned SOImmValV1 = 0, SOImmValV2 = 0;1082 1083 if (ARM_AM::isSOImmTwoPartVal(ImmVal)) { // Expand into a movi + orr.1084 LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);1085 HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))1086 .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))1087 .addReg(DstReg);1088 SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);1089 SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);1090 } else { // Expand into a mvn + sub.1091 LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MVNi), DstReg);1092 HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri))1093 .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))1094 .addReg(DstReg);1095 SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(-ImmVal);1096 SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(-ImmVal);1097 SOImmValV1 = ~(-SOImmValV1);1098 }1099 1100 unsigned MIFlags = MI.getFlags();1101 LO16 = LO16.addImm(SOImmValV1);1102 HI16 = HI16.addImm(SOImmValV2);1103 LO16.cloneMemRefs(MI);1104 HI16.cloneMemRefs(MI);1105 LO16.setMIFlags(MIFlags);1106 HI16.setMIFlags(MIFlags);1107 LO16.addImm(Pred).addReg(PredReg).add(condCodeOp());1108 HI16.addImm(Pred).addReg(PredReg).add(condCodeOp());1109 if (isCC)1110 LO16.add(makeImplicit(MI.getOperand(1)));1111 LO16.copyImplicitOps(MI);1112 HI16.copyImplicitOps(MI);1113 MI.eraseFromParent();1114 return;1115 }1116 1117 unsigned LO16Opc = 0;1118 unsigned HI16Opc = 0;1119 unsigned MIFlags = MI.getFlags();1120 if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {1121 LO16Opc = ARM::t2MOVi16;1122 HI16Opc = ARM::t2MOVTi16;1123 } else {1124 LO16Opc = ARM::MOVi16;1125 HI16Opc = ARM::MOVTi16;1126 }1127 1128 LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);1129 LO16.setMIFlags(MIFlags);1130 LO16.add(getMovOperand(MO, ARMII::MO_LO16));1131 LO16.cloneMemRefs(MI);1132 LO16.addImm(Pred).addReg(PredReg);1133 if (isCC)1134 LO16.add(makeImplicit(MI.getOperand(1)));1135 LO16.copyImplicitOps(MI);1136 LLVM_DEBUG(dbgs() << "To: "; LO16.getInstr()->dump(););1137 1138 MachineOperand HIOperand = getMovOperand(MO, ARMII::MO_HI16);1139 if (!(HIOperand.isImm() && HIOperand.getImm() == 0)) {1140 HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))1141 .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))1142 .addReg(DstReg);1143 HI16.setMIFlags(MIFlags);1144 HI16.add(HIOperand);1145 HI16.cloneMemRefs(MI);1146 HI16.addImm(Pred).addReg(PredReg);1147 HI16.copyImplicitOps(MI);1148 LLVM_DEBUG(dbgs() << "And: "; HI16.getInstr()->dump(););1149 } else {1150 LO16->getOperand(0).setIsDead(DstIsDead);1151 }1152 1153 if (RequiresBundling)1154 finalizeBundle(MBB, LO16->getIterator(), MBBI->getIterator());1155 1156 MI.eraseFromParent();1157}1158 1159// The size of the area, accessed by that VLSTM/VLLDM1160// S0-S31 + FPSCR + 8 more bytes (VPR + pad, or just pad)1161static const int CMSE_FP_SAVE_SIZE = 136;1162 1163static void determineGPRegsToClear(const MachineInstr &MI,1164 const std::initializer_list<unsigned> &Regs,1165 SmallVectorImpl<unsigned> &ClearRegs) {1166 SmallVector<unsigned, 4> OpRegs;1167 for (const MachineOperand &Op : MI.operands()) {1168 if (!Op.isReg() || !Op.isUse())1169 continue;1170 OpRegs.push_back(Op.getReg());1171 }1172 llvm::sort(OpRegs);1173 1174 std::set_difference(Regs.begin(), Regs.end(), OpRegs.begin(), OpRegs.end(),1175 std::back_inserter(ClearRegs));1176}1177 1178void ARMExpandPseudo::CMSEClearGPRegs(1179 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,1180 const DebugLoc &DL, const SmallVectorImpl<unsigned> &ClearRegs,1181 unsigned ClobberReg) {1182 1183 if (STI->hasV8_1MMainlineOps()) {1184 // Clear the registers using the CLRM instruction.1185 MachineInstrBuilder CLRM =1186 BuildMI(MBB, MBBI, DL, TII->get(ARM::t2CLRM)).add(predOps(ARMCC::AL));1187 for (unsigned R : ClearRegs)1188 CLRM.addReg(R, RegState::Define);1189 CLRM.addReg(ARM::APSR, RegState::Define);1190 CLRM.addReg(ARM::CPSR, RegState::Define | RegState::Implicit);1191 } else {1192 // Clear the registers and flags by copying ClobberReg into them.1193 // (Baseline can't do a high register clear in one instruction).1194 for (unsigned Reg : ClearRegs) {1195 if (Reg == ClobberReg)1196 continue;1197 BuildMI(MBB, MBBI, DL, TII->get(ARM::tMOVr), Reg)1198 .addReg(ClobberReg)1199 .add(predOps(ARMCC::AL));1200 }1201 1202 BuildMI(MBB, MBBI, DL, TII->get(ARM::t2MSR_M))1203 .addImm(STI->hasDSP() ? 0xc00 : 0x800)1204 .addReg(ClobberReg)1205 .add(predOps(ARMCC::AL));1206 }1207}1208 1209// Find which FP registers need to be cleared. The parameter `ClearRegs` is1210// initialised with all elements set to true, and this function resets all the1211// bits, which correspond to register uses. Returns true if any floating point1212// register is defined, false otherwise.1213static bool determineFPRegsToClear(const MachineInstr &MI,1214 BitVector &ClearRegs) {1215 bool DefFP = false;1216 for (const MachineOperand &Op : MI.operands()) {1217 if (!Op.isReg())1218 continue;1219 1220 Register Reg = Op.getReg();1221 if (Op.isDef()) {1222 if ((Reg >= ARM::Q0 && Reg <= ARM::Q7) ||1223 (Reg >= ARM::D0 && Reg <= ARM::D15) ||1224 (Reg >= ARM::S0 && Reg <= ARM::S31))1225 DefFP = true;1226 continue;1227 }1228 1229 if (Reg >= ARM::Q0 && Reg <= ARM::Q7) {1230 int R = Reg - ARM::Q0;1231 ClearRegs.reset(R * 4, (R + 1) * 4);1232 } else if (Reg >= ARM::D0 && Reg <= ARM::D15) {1233 int R = Reg - ARM::D0;1234 ClearRegs.reset(R * 2, (R + 1) * 2);1235 } else if (Reg >= ARM::S0 && Reg <= ARM::S31) {1236 ClearRegs[Reg - ARM::S0] = false;1237 }1238 }1239 return DefFP;1240}1241 1242MachineBasicBlock &1243ARMExpandPseudo::CMSEClearFPRegs(MachineBasicBlock &MBB,1244 MachineBasicBlock::iterator MBBI) {1245 BitVector ClearRegs(16, true);1246 (void)determineFPRegsToClear(*MBBI, ClearRegs);1247 1248 if (STI->hasV8_1MMainlineOps())1249 return CMSEClearFPRegsV81(MBB, MBBI, ClearRegs);1250 else1251 return CMSEClearFPRegsV8(MBB, MBBI, ClearRegs);1252}1253 1254// Clear the FP registers for v8.0-M, by copying over the content1255// of LR. Uses R12 as a scratch register.1256MachineBasicBlock &1257ARMExpandPseudo::CMSEClearFPRegsV8(MachineBasicBlock &MBB,1258 MachineBasicBlock::iterator MBBI,1259 const BitVector &ClearRegs) {1260 if (!STI->hasFPRegs())1261 return MBB;1262 1263 auto &RetI = *MBBI;1264 const DebugLoc &DL = RetI.getDebugLoc();1265 1266 // If optimising for minimum size, clear FP registers unconditionally.1267 // Otherwise, check the CONTROL.SFPA (Secure Floating-Point Active) bit and1268 // don't clear them if they belong to the non-secure state.1269 MachineBasicBlock *ClearBB, *DoneBB;1270 if (STI->hasMinSize()) {1271 ClearBB = DoneBB = &MBB;1272 } else {1273 MachineFunction *MF = MBB.getParent();1274 ClearBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());1275 DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());1276 1277 MF->insert(++MBB.getIterator(), ClearBB);1278 MF->insert(++ClearBB->getIterator(), DoneBB);1279 1280 DoneBB->splice(DoneBB->end(), &MBB, MBBI, MBB.end());1281 DoneBB->transferSuccessors(&MBB);1282 MBB.addSuccessor(ClearBB);1283 MBB.addSuccessor(DoneBB);1284 ClearBB->addSuccessor(DoneBB);1285 1286 // At the new basic blocks we need to have live-in the registers, used1287 // for the return value as well as LR, used to clear registers.1288 for (const MachineOperand &Op : RetI.operands()) {1289 if (!Op.isReg())1290 continue;1291 Register Reg = Op.getReg();1292 if (Reg == ARM::NoRegister || Reg == ARM::LR)1293 continue;1294 assert(Reg.isPhysical() && "Unallocated register");1295 ClearBB->addLiveIn(Reg);1296 DoneBB->addLiveIn(Reg);1297 }1298 ClearBB->addLiveIn(ARM::LR);1299 DoneBB->addLiveIn(ARM::LR);1300 1301 // Read the CONTROL register.1302 BuildMI(MBB, MBB.end(), DL, TII->get(ARM::t2MRS_M), ARM::R12)1303 .addImm(20)1304 .add(predOps(ARMCC::AL));1305 // Check bit 3 (SFPA).1306 BuildMI(MBB, MBB.end(), DL, TII->get(ARM::t2TSTri))1307 .addReg(ARM::R12)1308 .addImm(8)1309 .add(predOps(ARMCC::AL));1310 // If SFPA is clear, jump over ClearBB to DoneBB.1311 BuildMI(MBB, MBB.end(), DL, TII->get(ARM::tBcc))1312 .addMBB(DoneBB)1313 .addImm(ARMCC::EQ)1314 .addReg(ARM::CPSR, RegState::Kill);1315 }1316 1317 // Emit the clearing sequence1318 for (unsigned D = 0; D < 8; D++) {1319 // Attempt to clear as double1320 if (ClearRegs[D * 2 + 0] && ClearRegs[D * 2 + 1]) {1321 unsigned Reg = ARM::D0 + D;1322 BuildMI(ClearBB, DL, TII->get(ARM::VMOVDRR), Reg)1323 .addReg(ARM::LR)1324 .addReg(ARM::LR)1325 .add(predOps(ARMCC::AL));1326 } else {1327 // Clear first part as single1328 if (ClearRegs[D * 2 + 0]) {1329 unsigned Reg = ARM::S0 + D * 2;1330 BuildMI(ClearBB, DL, TII->get(ARM::VMOVSR), Reg)1331 .addReg(ARM::LR)1332 .add(predOps(ARMCC::AL));1333 }1334 // Clear second part as single1335 if (ClearRegs[D * 2 + 1]) {1336 unsigned Reg = ARM::S0 + D * 2 + 1;1337 BuildMI(ClearBB, DL, TII->get(ARM::VMOVSR), Reg)1338 .addReg(ARM::LR)1339 .add(predOps(ARMCC::AL));1340 }1341 }1342 }1343 1344 // Clear FPSCR bits 0-4, 7, 28-311345 // The other bits are program global according to the AAPCS1346 BuildMI(ClearBB, DL, TII->get(ARM::VMRS), ARM::R12)1347 .add(predOps(ARMCC::AL));1348 BuildMI(ClearBB, DL, TII->get(ARM::t2BICri), ARM::R12)1349 .addReg(ARM::R12)1350 .addImm(0x0000009F)1351 .add(predOps(ARMCC::AL))1352 .add(condCodeOp());1353 BuildMI(ClearBB, DL, TII->get(ARM::t2BICri), ARM::R12)1354 .addReg(ARM::R12)1355 .addImm(0xF0000000)1356 .add(predOps(ARMCC::AL))1357 .add(condCodeOp());1358 BuildMI(ClearBB, DL, TII->get(ARM::VMSR))1359 .addReg(ARM::R12)1360 .add(predOps(ARMCC::AL));1361 1362 return *DoneBB;1363}1364 1365MachineBasicBlock &1366ARMExpandPseudo::CMSEClearFPRegsV81(MachineBasicBlock &MBB,1367 MachineBasicBlock::iterator MBBI,1368 const BitVector &ClearRegs) {1369 auto &RetI = *MBBI;1370 1371 // Emit a sequence of VSCCLRM <sreglist> instructions, one instruction for1372 // each contiguous sequence of S-registers.1373 int Start = -1, End = -1;1374 for (int S = 0, E = ClearRegs.size(); S != E; ++S) {1375 if (ClearRegs[S] && S == End + 1) {1376 End = S; // extend range1377 continue;1378 }1379 // Emit current range.1380 if (Start < End) {1381 MachineInstrBuilder VSCCLRM =1382 BuildMI(MBB, MBBI, RetI.getDebugLoc(), TII->get(ARM::VSCCLRMS))1383 .add(predOps(ARMCC::AL));1384 while (++Start <= End)1385 VSCCLRM.addReg(ARM::S0 + Start, RegState::Define);1386 VSCCLRM.addReg(ARM::VPR, RegState::Define);1387 }1388 Start = End = S;1389 }1390 // Emit last range.1391 if (Start < End) {1392 MachineInstrBuilder VSCCLRM =1393 BuildMI(MBB, MBBI, RetI.getDebugLoc(), TII->get(ARM::VSCCLRMS))1394 .add(predOps(ARMCC::AL));1395 while (++Start <= End)1396 VSCCLRM.addReg(ARM::S0 + Start, RegState::Define);1397 VSCCLRM.addReg(ARM::VPR, RegState::Define);1398 }1399 1400 return MBB;1401}1402 1403void ARMExpandPseudo::CMSESaveClearFPRegs(1404 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,1405 const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {1406 if (STI->hasV8_1MMainlineOps())1407 CMSESaveClearFPRegsV81(MBB, MBBI, DL, LiveRegs);1408 else if (STI->hasV8MMainlineOps())1409 CMSESaveClearFPRegsV8(MBB, MBBI, DL, LiveRegs, ScratchRegs);1410}1411 1412// Save and clear FP registers if present1413void ARMExpandPseudo::CMSESaveClearFPRegsV8(1414 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,1415 const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {1416 1417 // Store an available register for FPSCR clearing1418 assert(!ScratchRegs.empty());1419 unsigned SpareReg = ScratchRegs.front();1420 1421 // save space on stack for VLSTM1422 BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBspi), ARM::SP)1423 .addReg(ARM::SP)1424 .addImm(CMSE_FP_SAVE_SIZE >> 2)1425 .add(predOps(ARMCC::AL));1426 1427 // Use ScratchRegs to store the fp regs1428 std::vector<std::tuple<unsigned, unsigned, unsigned>> ClearedFPRegs;1429 std::vector<unsigned> NonclearedFPRegs;1430 bool ReturnsFPReg = false;1431 for (const MachineOperand &Op : MBBI->operands()) {1432 if (Op.isReg() && Op.isUse()) {1433 Register Reg = Op.getReg();1434 assert(!ARM::DPRRegClass.contains(Reg) ||1435 ARM::DPR_VFP2RegClass.contains(Reg));1436 assert(!ARM::QPRRegClass.contains(Reg));1437 if (ARM::DPR_VFP2RegClass.contains(Reg)) {1438 if (ScratchRegs.size() >= 2) {1439 unsigned SaveReg2 = ScratchRegs.pop_back_val();1440 unsigned SaveReg1 = ScratchRegs.pop_back_val();1441 ClearedFPRegs.emplace_back(Reg, SaveReg1, SaveReg2);1442 1443 // Save the fp register to the normal registers1444 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRRD))1445 .addReg(SaveReg1, RegState::Define)1446 .addReg(SaveReg2, RegState::Define)1447 .addReg(Reg)1448 .add(predOps(ARMCC::AL));1449 } else {1450 NonclearedFPRegs.push_back(Reg);1451 }1452 } else if (ARM::SPRRegClass.contains(Reg)) {1453 if (ScratchRegs.size() >= 1) {1454 unsigned SaveReg = ScratchRegs.pop_back_val();1455 ClearedFPRegs.emplace_back(Reg, SaveReg, 0);1456 1457 // Save the fp register to the normal registers1458 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRS), SaveReg)1459 .addReg(Reg)1460 .add(predOps(ARMCC::AL));1461 } else {1462 NonclearedFPRegs.push_back(Reg);1463 }1464 }1465 } else if (Op.isReg() && Op.isDef()) {1466 Register Reg = Op.getReg();1467 if (ARM::SPRRegClass.contains(Reg) || ARM::DPRRegClass.contains(Reg) ||1468 ARM::QPRRegClass.contains(Reg))1469 ReturnsFPReg = true;1470 }1471 }1472 1473 bool PassesFPReg = (!NonclearedFPRegs.empty() || !ClearedFPRegs.empty());1474 1475 if (PassesFPReg || ReturnsFPReg)1476 assert(STI->hasFPRegs() && "Subtarget needs fpregs");1477 1478 // CVE-2024-78831479 //1480 // The VLLDM/VLSTM instructions set up lazy state preservation, but they1481 // execute as NOPs if the FP register file is not considered to contain1482 // secure data, represented by the CONTROL_S.SFPA bit. This means that the1483 // state of CONTROL_S.SFPA must be the same when these two instructions are1484 // executed. That might not be the case if we haven't used any FP1485 // instructions before the VLSTM, so CONTROL_S.SFPA is clear, but do have one1486 // before the VLLDM, which sets it..1487 //1488 // If we can't prove that SFPA will be the same for the VLSTM and VLLDM, we1489 // execute a "vmov s0, s0" instruction before the VLSTM to ensure that1490 // CONTROL_S.SFPA is set for both.1491 //1492 // That can only happen for callees which take no FP arguments (or we'd have1493 // inserted a VMOV above) and which return values in FP regs (so that we need1494 // to use a VMOV to back-up the return value before the VLLDM). It also can't1495 // happen if the call is dominated by other existing floating-point1496 // instructions, but we don't currently check for that case.1497 //1498 // These conditions mean that we only emit this instruction when using the1499 // hard-float ABI, which means we can assume that FP instructions are1500 // available, and don't need to make it conditional like we do for the1501 // CVE-2021-35465 workaround.1502 if (ReturnsFPReg && !PassesFPReg) {1503 bool S0Dead = !LiveRegs.contains(ARM::S0);1504 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVS))1505 .addReg(ARM::S0, RegState::Define | getDeadRegState(S0Dead))1506 .addReg(ARM::S0, getUndefRegState(S0Dead))1507 .add(predOps(ARMCC::AL));1508 }1509 1510 // Lazy store all fp registers to the stack.1511 // This executes as NOP in the absence of floating-point support.1512 MachineInstrBuilder VLSTM =1513 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM))1514 .addReg(ARM::SP)1515 .add(predOps(ARMCC::AL))1516 .addImm(0); // Represents a pseoudo register list, has no effect on1517 // the encoding.1518 // Mark non-live registers as undef1519 for (MachineOperand &MO : VLSTM->implicit_operands()) {1520 if (MO.isReg() && !MO.isDef()) {1521 Register Reg = MO.getReg();1522 MO.setIsUndef(!LiveRegs.contains(Reg));1523 }1524 }1525 1526 // Restore all arguments1527 for (const auto &Regs : ClearedFPRegs) {1528 unsigned Reg, SaveReg1, SaveReg2;1529 std::tie(Reg, SaveReg1, SaveReg2) = Regs;1530 if (ARM::DPR_VFP2RegClass.contains(Reg))1531 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVDRR), Reg)1532 .addReg(SaveReg1)1533 .addReg(SaveReg2)1534 .add(predOps(ARMCC::AL));1535 else if (ARM::SPRRegClass.contains(Reg))1536 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVSR), Reg)1537 .addReg(SaveReg1)1538 .add(predOps(ARMCC::AL));1539 }1540 1541 for (unsigned Reg : NonclearedFPRegs) {1542 if (ARM::DPR_VFP2RegClass.contains(Reg)) {1543 if (STI->isLittle()) {1544 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRD), Reg)1545 .addReg(ARM::SP)1546 .addImm((Reg - ARM::D0) * 2)1547 .add(predOps(ARMCC::AL));1548 } else {1549 // For big-endian targets we need to load the two subregisters of Reg1550 // manually because VLDRD would load them in wrong order1551 MCRegister SReg0 = TRI->getSubReg(Reg, ARM::ssub_0);1552 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRS), SReg0)1553 .addReg(ARM::SP)1554 .addImm((Reg - ARM::D0) * 2)1555 .add(predOps(ARMCC::AL));1556 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRS), SReg0 + 1)1557 .addReg(ARM::SP)1558 .addImm((Reg - ARM::D0) * 2 + 1)1559 .add(predOps(ARMCC::AL));1560 }1561 } else if (ARM::SPRRegClass.contains(Reg)) {1562 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRS), Reg)1563 .addReg(ARM::SP)1564 .addImm(Reg - ARM::S0)1565 .add(predOps(ARMCC::AL));1566 }1567 }1568 // restore FPSCR from stack and clear bits 0-4, 7, 28-311569 // The other bits are program global according to the AAPCS1570 if (PassesFPReg) {1571 BuildMI(MBB, MBBI, DL, TII->get(ARM::tLDRspi), SpareReg)1572 .addReg(ARM::SP)1573 .addImm(0x10)1574 .add(predOps(ARMCC::AL));1575 BuildMI(MBB, MBBI, DL, TII->get(ARM::t2BICri), SpareReg)1576 .addReg(SpareReg)1577 .addImm(0x0000009F)1578 .add(predOps(ARMCC::AL))1579 .add(condCodeOp());1580 BuildMI(MBB, MBBI, DL, TII->get(ARM::t2BICri), SpareReg)1581 .addReg(SpareReg)1582 .addImm(0xF0000000)1583 .add(predOps(ARMCC::AL))1584 .add(condCodeOp());1585 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMSR))1586 .addReg(SpareReg)1587 .add(predOps(ARMCC::AL));1588 // The ldr must happen after a floating point instruction. To prevent the1589 // post-ra scheduler to mess with the order, we create a bundle.1590 finalizeBundle(MBB, VLSTM->getIterator(), MBBI->getIterator());1591 }1592}1593 1594void ARMExpandPseudo::CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,1595 MachineBasicBlock::iterator MBBI,1596 DebugLoc &DL,1597 const LivePhysRegs &LiveRegs) {1598 BitVector ClearRegs(32, true);1599 bool DefFP = determineFPRegsToClear(*MBBI, ClearRegs);1600 1601 // If the instruction does not write to a FP register and no elements were1602 // removed from the set, then no FP registers were used to pass1603 // arguments/returns.1604 if (!DefFP && ClearRegs.count() == ClearRegs.size()) {1605 // save space on stack for VLSTM1606 BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBspi), ARM::SP)1607 .addReg(ARM::SP)1608 .addImm(CMSE_FP_SAVE_SIZE >> 2)1609 .add(predOps(ARMCC::AL));1610 1611 // Lazy store all FP registers to the stack1612 MachineInstrBuilder VLSTM =1613 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM))1614 .addReg(ARM::SP)1615 .add(predOps(ARMCC::AL))1616 .addImm(0); // Represents a pseoudo register list, has no effect on1617 // the encoding.1618 // Mark non-live registers as undef1619 for (MachineOperand &MO : VLSTM->implicit_operands()) {1620 if (MO.isReg() && !MO.isDef()) {1621 Register Reg = MO.getReg();1622 MO.setIsUndef(!LiveRegs.contains(Reg));1623 }1624 }1625 } else {1626 // Push all the callee-saved registers (s16-s31).1627 MachineInstrBuilder VPUSH =1628 BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTMSDB_UPD), ARM::SP)1629 .addReg(ARM::SP)1630 .add(predOps(ARMCC::AL));1631 for (unsigned Reg = ARM::S16; Reg <= ARM::S31; ++Reg)1632 VPUSH.addReg(Reg);1633 1634 // Clear FP registers with a VSCCLRM.1635 (void)CMSEClearFPRegsV81(MBB, MBBI, ClearRegs);1636 1637 // Save floating-point context.1638 BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTR_FPCXTS_pre), ARM::SP)1639 .addReg(ARM::SP)1640 .addImm(-8)1641 .add(predOps(ARMCC::AL));1642 }1643}1644 1645// Restore FP registers if present1646void ARMExpandPseudo::CMSERestoreFPRegs(1647 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,1648 SmallVectorImpl<unsigned> &AvailableRegs) {1649 if (STI->hasV8_1MMainlineOps())1650 CMSERestoreFPRegsV81(MBB, MBBI, DL, AvailableRegs);1651 else if (STI->hasV8MMainlineOps())1652 CMSERestoreFPRegsV8(MBB, MBBI, DL, AvailableRegs);1653}1654 1655void ARMExpandPseudo::CMSERestoreFPRegsV8(1656 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,1657 SmallVectorImpl<unsigned> &AvailableRegs) {1658 1659 // Keep a scratch register for the mitigation sequence.1660 unsigned ScratchReg = ARM::NoRegister;1661 if (STI->fixCMSE_CVE_2021_35465())1662 ScratchReg = AvailableRegs.pop_back_val();1663 1664 // Use AvailableRegs to store the fp regs1665 std::vector<std::tuple<unsigned, unsigned, unsigned>> ClearedFPRegs;1666 std::vector<unsigned> NonclearedFPRegs;1667 for (const MachineOperand &Op : MBBI->operands()) {1668 if (Op.isReg() && Op.isDef()) {1669 Register Reg = Op.getReg();1670 assert(!ARM::DPRRegClass.contains(Reg) ||1671 ARM::DPR_VFP2RegClass.contains(Reg));1672 assert(!ARM::QPRRegClass.contains(Reg));1673 if (ARM::DPR_VFP2RegClass.contains(Reg)) {1674 if (AvailableRegs.size() >= 2) {1675 unsigned SaveReg2 = AvailableRegs.pop_back_val();1676 unsigned SaveReg1 = AvailableRegs.pop_back_val();1677 ClearedFPRegs.emplace_back(Reg, SaveReg1, SaveReg2);1678 1679 // Save the fp register to the normal registers1680 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRRD))1681 .addReg(SaveReg1, RegState::Define)1682 .addReg(SaveReg2, RegState::Define)1683 .addReg(Reg)1684 .add(predOps(ARMCC::AL));1685 } else {1686 NonclearedFPRegs.push_back(Reg);1687 }1688 } else if (ARM::SPRRegClass.contains(Reg)) {1689 if (AvailableRegs.size() >= 1) {1690 unsigned SaveReg = AvailableRegs.pop_back_val();1691 ClearedFPRegs.emplace_back(Reg, SaveReg, 0);1692 1693 // Save the fp register to the normal registers1694 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRS), SaveReg)1695 .addReg(Reg)1696 .add(predOps(ARMCC::AL));1697 } else {1698 NonclearedFPRegs.push_back(Reg);1699 }1700 }1701 }1702 }1703 1704 bool returnsFPReg = (!NonclearedFPRegs.empty() || !ClearedFPRegs.empty());1705 1706 if (returnsFPReg)1707 assert(STI->hasFPRegs() && "Subtarget needs fpregs");1708 1709 // Push FP regs that cannot be restored via normal registers on the stack1710 for (unsigned Reg : NonclearedFPRegs) {1711 if (ARM::DPR_VFP2RegClass.contains(Reg))1712 BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTRD))1713 .addReg(Reg)1714 .addReg(ARM::SP)1715 .addImm((Reg - ARM::D0) * 2)1716 .add(predOps(ARMCC::AL));1717 else if (ARM::SPRRegClass.contains(Reg))1718 BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTRS))1719 .addReg(Reg)1720 .addReg(ARM::SP)1721 .addImm(Reg - ARM::S0)1722 .add(predOps(ARMCC::AL));1723 }1724 1725 // Lazy load fp regs from stack.1726 // This executes as NOP in the absence of floating-point support.1727 MachineInstrBuilder VLLDM =1728 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLLDM))1729 .addReg(ARM::SP)1730 .add(predOps(ARMCC::AL))1731 .addImm(0); // Represents a pseoudo register list, has no effect on1732 // the encoding.1733 1734 if (STI->fixCMSE_CVE_2021_35465()) {1735 auto Bundler = MIBundleBuilder(MBB, VLLDM);1736 // Read the CONTROL register.1737 Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::t2MRS_M))1738 .addReg(ScratchReg, RegState::Define)1739 .addImm(20)1740 .add(predOps(ARMCC::AL)));1741 // Check bit 3 (SFPA).1742 Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::t2TSTri))1743 .addReg(ScratchReg)1744 .addImm(8)1745 .add(predOps(ARMCC::AL)));1746 // Emit the IT block.1747 Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::t2IT))1748 .addImm(ARMCC::NE)1749 .addImm(8));1750 // If SFPA is clear jump over to VLLDM, otherwise execute an instruction1751 // which has no functional effect apart from causing context creation:1752 // vmovne s0, s0. In the absence of FPU we emit .inst.w 0xeeb00a40,1753 // which is defined as NOP if not executed.1754 if (STI->hasFPRegs())1755 Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::VMOVS))1756 .addReg(ARM::S0, RegState::Define)1757 .addReg(ARM::S0, RegState::Undef)1758 .add(predOps(ARMCC::NE)));1759 else1760 Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::INLINEASM))1761 .addExternalSymbol(".inst.w 0xeeb00a40")1762 .addImm(InlineAsm::Extra_HasSideEffects));1763 finalizeBundle(MBB, Bundler.begin(), Bundler.end());1764 }1765 1766 // Restore all FP registers via normal registers1767 for (const auto &Regs : ClearedFPRegs) {1768 unsigned Reg, SaveReg1, SaveReg2;1769 std::tie(Reg, SaveReg1, SaveReg2) = Regs;1770 if (ARM::DPR_VFP2RegClass.contains(Reg))1771 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVDRR), Reg)1772 .addReg(SaveReg1)1773 .addReg(SaveReg2)1774 .add(predOps(ARMCC::AL));1775 else if (ARM::SPRRegClass.contains(Reg))1776 BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVSR), Reg)1777 .addReg(SaveReg1)1778 .add(predOps(ARMCC::AL));1779 }1780 1781 // Pop the stack space1782 BuildMI(MBB, MBBI, DL, TII->get(ARM::tADDspi), ARM::SP)1783 .addReg(ARM::SP)1784 .addImm(CMSE_FP_SAVE_SIZE >> 2)1785 .add(predOps(ARMCC::AL));1786}1787 1788static bool definesOrUsesFPReg(const MachineInstr &MI) {1789 for (const MachineOperand &Op : MI.operands()) {1790 if (!Op.isReg())1791 continue;1792 Register Reg = Op.getReg();1793 if ((Reg >= ARM::Q0 && Reg <= ARM::Q7) ||1794 (Reg >= ARM::D0 && Reg <= ARM::D15) ||1795 (Reg >= ARM::S0 && Reg <= ARM::S31))1796 return true;1797 }1798 return false;1799}1800 1801void ARMExpandPseudo::CMSERestoreFPRegsV81(1802 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,1803 SmallVectorImpl<unsigned> &AvailableRegs) {1804 if (!definesOrUsesFPReg(*MBBI)) {1805 if (STI->fixCMSE_CVE_2021_35465()) {1806 BuildMI(MBB, MBBI, DL, TII->get(ARM::VSCCLRMS))1807 .add(predOps(ARMCC::AL))1808 .addReg(ARM::VPR, RegState::Define);1809 }1810 1811 // Load FP registers from stack.1812 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLLDM))1813 .addReg(ARM::SP)1814 .add(predOps(ARMCC::AL))1815 .addImm(0); // Represents a pseoudo register list, has no effect on the1816 // encoding.1817 1818 // Pop the stack space1819 BuildMI(MBB, MBBI, DL, TII->get(ARM::tADDspi), ARM::SP)1820 .addReg(ARM::SP)1821 .addImm(CMSE_FP_SAVE_SIZE >> 2)1822 .add(predOps(ARMCC::AL));1823 } else {1824 // Restore the floating point context.1825 BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(ARM::VLDR_FPCXTS_post),1826 ARM::SP)1827 .addReg(ARM::SP)1828 .addImm(8)1829 .add(predOps(ARMCC::AL));1830 1831 // Pop all the callee-saved registers (s16-s31).1832 MachineInstrBuilder VPOP =1833 BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDMSIA_UPD), ARM::SP)1834 .addReg(ARM::SP)1835 .add(predOps(ARMCC::AL));1836 for (unsigned Reg = ARM::S16; Reg <= ARM::S31; ++Reg)1837 VPOP.addReg(Reg, RegState::Define);1838 }1839}1840 1841/// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as1842/// possible. This only gets used at -O0 so we don't care about efficiency of1843/// the generated code.1844bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,1845 MachineBasicBlock::iterator MBBI,1846 unsigned LdrexOp, unsigned StrexOp,1847 unsigned UxtOp,1848 MachineBasicBlock::iterator &NextMBBI) {1849 bool IsThumb = STI->isThumb();1850 bool IsThumb1Only = STI->isThumb1Only();1851 MachineInstr &MI = *MBBI;1852 DebugLoc DL = MI.getDebugLoc();1853 const MachineOperand &Dest = MI.getOperand(0);1854 Register TempReg = MI.getOperand(1).getReg();1855 // Duplicating undef operands into 2 instructions does not guarantee the same1856 // value on both; However undef should be replaced by xzr anyway.1857 assert(!MI.getOperand(2).isUndef() && "cannot handle undef");1858 Register AddrReg = MI.getOperand(2).getReg();1859 Register DesiredReg = MI.getOperand(3).getReg();1860 Register NewReg = MI.getOperand(4).getReg();1861 1862 if (IsThumb) {1863 assert(STI->hasV8MBaselineOps() &&1864 "CMP_SWAP not expected to be custom expanded for Thumb1");1865 assert((UxtOp == 0 || UxtOp == ARM::tUXTB || UxtOp == ARM::tUXTH) &&1866 "ARMv8-M.baseline does not have t2UXTB/t2UXTH");1867 assert((UxtOp == 0 || ARM::tGPRRegClass.contains(DesiredReg)) &&1868 "DesiredReg used for UXT op must be tGPR");1869 }1870 1871 MachineFunction *MF = MBB.getParent();1872 auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());1873 auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());1874 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());1875 1876 MF->insert(++MBB.getIterator(), LoadCmpBB);1877 MF->insert(++LoadCmpBB->getIterator(), StoreBB);1878 MF->insert(++StoreBB->getIterator(), DoneBB);1879 1880 if (UxtOp) {1881 MachineInstrBuilder MIB =1882 BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)1883 .addReg(DesiredReg, RegState::Kill);1884 if (!IsThumb)1885 MIB.addImm(0);1886 MIB.add(predOps(ARMCC::AL));1887 }1888 1889 // .Lloadcmp:1890 // ldrex rDest, [rAddr]1891 // cmp rDest, rDesired1892 // bne .Ldone1893 1894 MachineInstrBuilder MIB;1895 MIB = BuildMI(LoadCmpBB, DL, TII->get(LdrexOp), Dest.getReg());1896 MIB.addReg(AddrReg);1897 if (LdrexOp == ARM::t2LDREX)1898 MIB.addImm(0); // a 32-bit Thumb ldrex (only) allows an offset.1899 MIB.add(predOps(ARMCC::AL));1900 1901 unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;1902 BuildMI(LoadCmpBB, DL, TII->get(CMPrr))1903 .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))1904 .addReg(DesiredReg)1905 .add(predOps(ARMCC::AL));1906 unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;1907 BuildMI(LoadCmpBB, DL, TII->get(Bcc))1908 .addMBB(DoneBB)1909 .addImm(ARMCC::NE)1910 .addReg(ARM::CPSR, RegState::Kill);1911 LoadCmpBB->addSuccessor(DoneBB);1912 LoadCmpBB->addSuccessor(StoreBB);1913 1914 // .Lstore:1915 // strex rTempReg, rNew, [rAddr]1916 // cmp rTempReg, #01917 // bne .Lloadcmp1918 MIB = BuildMI(StoreBB, DL, TII->get(StrexOp), TempReg)1919 .addReg(NewReg)1920 .addReg(AddrReg);1921 if (StrexOp == ARM::t2STREX)1922 MIB.addImm(0); // a 32-bit Thumb strex (only) allows an offset.1923 MIB.add(predOps(ARMCC::AL));1924 1925 unsigned CMPri =1926 IsThumb ? (IsThumb1Only ? ARM::tCMPi8 : ARM::t2CMPri) : ARM::CMPri;1927 BuildMI(StoreBB, DL, TII->get(CMPri))1928 .addReg(TempReg, RegState::Kill)1929 .addImm(0)1930 .add(predOps(ARMCC::AL));1931 BuildMI(StoreBB, DL, TII->get(Bcc))1932 .addMBB(LoadCmpBB)1933 .addImm(ARMCC::NE)1934 .addReg(ARM::CPSR, RegState::Kill);1935 StoreBB->addSuccessor(LoadCmpBB);1936 StoreBB->addSuccessor(DoneBB);1937 1938 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());1939 DoneBB->transferSuccessors(&MBB);1940 1941 MBB.addSuccessor(LoadCmpBB);1942 1943 NextMBBI = MBB.end();1944 MI.eraseFromParent();1945 1946 // Recompute livein lists.1947 LivePhysRegs LiveRegs;1948 computeAndAddLiveIns(LiveRegs, *DoneBB);1949 computeAndAddLiveIns(LiveRegs, *StoreBB);1950 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);1951 // Do an extra pass around the loop to get loop carried registers right.1952 StoreBB->clearLiveIns();1953 computeAndAddLiveIns(LiveRegs, *StoreBB);1954 LoadCmpBB->clearLiveIns();1955 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);1956 1957 return true;1958}1959 1960/// ARM's ldrexd/strexd take a consecutive register pair (represented as a1961/// single GPRPair register), Thumb's take two separate registers so we need to1962/// extract the subregs from the pair.1963static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,1964 unsigned Flags, bool IsThumb,1965 const TargetRegisterInfo *TRI) {1966 if (IsThumb) {1967 Register RegLo = TRI->getSubReg(Reg.getReg(), ARM::gsub_0);1968 Register RegHi = TRI->getSubReg(Reg.getReg(), ARM::gsub_1);1969 MIB.addReg(RegLo, Flags);1970 MIB.addReg(RegHi, Flags);1971 } else1972 MIB.addReg(Reg.getReg(), Flags);1973}1974 1975/// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.1976bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,1977 MachineBasicBlock::iterator MBBI,1978 MachineBasicBlock::iterator &NextMBBI) {1979 bool IsThumb = STI->isThumb();1980 assert(!STI->isThumb1Only() && "CMP_SWAP_64 unsupported under Thumb1!");1981 MachineInstr &MI = *MBBI;1982 DebugLoc DL = MI.getDebugLoc();1983 MachineOperand &Dest = MI.getOperand(0);1984 // Duplicating undef operands into 2 instructions does not guarantee the same1985 // value on both; However undef should be replaced by xzr anyway.1986 assert(!MI.getOperand(1).isUndef() && "cannot handle undef");1987 Register AddrAndTempReg = MI.getOperand(1).getReg();1988 Register AddrReg = TRI->getSubReg(AddrAndTempReg, ARM::gsub_0);1989 Register TempReg = TRI->getSubReg(AddrAndTempReg, ARM::gsub_1);1990 assert(MI.getOperand(1).getReg() == MI.getOperand(2).getReg() &&1991 "tied operands have different registers");1992 Register DesiredReg = MI.getOperand(3).getReg();1993 MachineOperand New = MI.getOperand(4);1994 New.setIsKill(false);1995 1996 Register DestLo = TRI->getSubReg(Dest.getReg(), ARM::gsub_0);1997 Register DestHi = TRI->getSubReg(Dest.getReg(), ARM::gsub_1);1998 Register DesiredLo = TRI->getSubReg(DesiredReg, ARM::gsub_0);1999 Register DesiredHi = TRI->getSubReg(DesiredReg, ARM::gsub_1);2000 2001 MachineFunction *MF = MBB.getParent();2002 auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());2003 auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());2004 auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());2005 2006 MF->insert(++MBB.getIterator(), LoadCmpBB);2007 MF->insert(++LoadCmpBB->getIterator(), StoreBB);2008 MF->insert(++StoreBB->getIterator(), DoneBB);2009 2010 // .Lloadcmp:2011 // ldrexd rDestLo, rDestHi, [rAddr]2012 // cmp rDestLo, rDesiredLo2013 // sbcs dead rTempReg, rDestHi, rDesiredHi2014 // bne .Ldone2015 unsigned LDREXD = IsThumb ? ARM::t2LDREXD : ARM::LDREXD;2016 MachineInstrBuilder MIB;2017 MIB = BuildMI(LoadCmpBB, DL, TII->get(LDREXD));2018 addExclusiveRegPair(MIB, Dest, RegState::Define, IsThumb, TRI);2019 MIB.addReg(AddrReg).add(predOps(ARMCC::AL));2020 2021 unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;2022 BuildMI(LoadCmpBB, DL, TII->get(CMPrr))2023 .addReg(DestLo, getKillRegState(Dest.isDead()))2024 .addReg(DesiredLo)2025 .add(predOps(ARMCC::AL));2026 2027 BuildMI(LoadCmpBB, DL, TII->get(CMPrr))2028 .addReg(DestHi, getKillRegState(Dest.isDead()))2029 .addReg(DesiredHi)2030 .addImm(ARMCC::EQ).addReg(ARM::CPSR, RegState::Kill);2031 2032 unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;2033 BuildMI(LoadCmpBB, DL, TII->get(Bcc))2034 .addMBB(DoneBB)2035 .addImm(ARMCC::NE)2036 .addReg(ARM::CPSR, RegState::Kill);2037 LoadCmpBB->addSuccessor(DoneBB);2038 LoadCmpBB->addSuccessor(StoreBB);2039 2040 // .Lstore:2041 // strexd rTempReg, rNewLo, rNewHi, [rAddr]2042 // cmp rTempReg, #02043 // bne .Lloadcmp2044 unsigned STREXD = IsThumb ? ARM::t2STREXD : ARM::STREXD;2045 MIB = BuildMI(StoreBB, DL, TII->get(STREXD), TempReg);2046 unsigned Flags = getKillRegState(New.isDead());2047 addExclusiveRegPair(MIB, New, Flags, IsThumb, TRI);2048 MIB.addReg(AddrReg).add(predOps(ARMCC::AL));2049 2050 unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;2051 BuildMI(StoreBB, DL, TII->get(CMPri))2052 .addReg(TempReg, RegState::Kill)2053 .addImm(0)2054 .add(predOps(ARMCC::AL));2055 BuildMI(StoreBB, DL, TII->get(Bcc))2056 .addMBB(LoadCmpBB)2057 .addImm(ARMCC::NE)2058 .addReg(ARM::CPSR, RegState::Kill);2059 StoreBB->addSuccessor(LoadCmpBB);2060 StoreBB->addSuccessor(DoneBB);2061 2062 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());2063 DoneBB->transferSuccessors(&MBB);2064 2065 MBB.addSuccessor(LoadCmpBB);2066 2067 NextMBBI = MBB.end();2068 MI.eraseFromParent();2069 2070 // Recompute livein lists.2071 LivePhysRegs LiveRegs;2072 computeAndAddLiveIns(LiveRegs, *DoneBB);2073 computeAndAddLiveIns(LiveRegs, *StoreBB);2074 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);2075 // Do an extra pass around the loop to get loop carried registers right.2076 StoreBB->clearLiveIns();2077 computeAndAddLiveIns(LiveRegs, *StoreBB);2078 LoadCmpBB->clearLiveIns();2079 computeAndAddLiveIns(LiveRegs, *LoadCmpBB);2080 2081 return true;2082}2083 2084static void CMSEPushCalleeSaves(const TargetInstrInfo &TII,2085 MachineBasicBlock &MBB,2086 MachineBasicBlock::iterator MBBI,2087 Register JumpReg, const LivePhysRegs &LiveRegs,2088 bool Thumb1Only) {2089 const DebugLoc &DL = MBBI->getDebugLoc();2090 if (Thumb1Only) { // push Lo and Hi regs separately2091 MachineInstrBuilder PushMIB =2092 BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH)).add(predOps(ARMCC::AL));2093 for (unsigned Reg = ARM::R4; Reg < ARM::R8; ++Reg) {2094 PushMIB.addReg(2095 Reg, Reg == JumpReg || LiveRegs.contains(Reg) ? 0 : RegState::Undef);2096 }2097 2098 // Thumb1 can only tPUSH low regs, so we copy the high regs to the low2099 // regs that we just saved and push the low regs again, taking care to2100 // not clobber JumpReg. If JumpReg is one of the low registers, push first2101 // the values of r9-r11, and then r8. That would leave them ordered in2102 // memory, and allow us to later pop them with a single instructions.2103 // FIXME: Could also use any of r0-r3 that are free (including in the2104 // first PUSH above).2105 for (unsigned LoReg = ARM::R7, HiReg = ARM::R11; LoReg >= ARM::R4;2106 --LoReg) {2107 if (JumpReg == LoReg)2108 continue;2109 BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), LoReg)2110 .addReg(HiReg, LiveRegs.contains(HiReg) ? 0 : RegState::Undef)2111 .add(predOps(ARMCC::AL));2112 --HiReg;2113 }2114 MachineInstrBuilder PushMIB2 =2115 BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH)).add(predOps(ARMCC::AL));2116 for (unsigned Reg = ARM::R4; Reg < ARM::R8; ++Reg) {2117 if (Reg == JumpReg)2118 continue;2119 PushMIB2.addReg(Reg, RegState::Kill);2120 }2121 2122 // If we couldn't use a low register for temporary storage (because it was2123 // the JumpReg), use r4 or r5, whichever is not JumpReg. It has already been2124 // saved.2125 if (JumpReg >= ARM::R4 && JumpReg <= ARM::R7) {2126 Register LoReg = JumpReg == ARM::R4 ? ARM::R5 : ARM::R4;2127 BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), LoReg)2128 .addReg(ARM::R8, LiveRegs.contains(ARM::R8) ? 0 : RegState::Undef)2129 .add(predOps(ARMCC::AL));2130 BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH))2131 .add(predOps(ARMCC::AL))2132 .addReg(LoReg, RegState::Kill);2133 }2134 } else { // push Lo and Hi registers with a single instruction2135 MachineInstrBuilder PushMIB =2136 BuildMI(MBB, MBBI, DL, TII.get(ARM::t2STMDB_UPD), ARM::SP)2137 .addReg(ARM::SP)2138 .add(predOps(ARMCC::AL));2139 for (unsigned Reg = ARM::R4; Reg < ARM::R12; ++Reg) {2140 PushMIB.addReg(2141 Reg, Reg == JumpReg || LiveRegs.contains(Reg) ? 0 : RegState::Undef);2142 }2143 }2144}2145 2146static void CMSEPopCalleeSaves(const TargetInstrInfo &TII,2147 MachineBasicBlock &MBB,2148 MachineBasicBlock::iterator MBBI,2149 bool Thumb1Only) {2150 const DebugLoc &DL = MBBI->getDebugLoc();2151 if (Thumb1Only) {2152 MachineInstrBuilder PopMIB =2153 BuildMI(MBB, MBBI, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL));2154 for (int R = 0; R < 4; ++R) {2155 PopMIB.addReg(ARM::R4 + R, RegState::Define);2156 BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), ARM::R8 + R)2157 .addReg(ARM::R4 + R, RegState::Kill)2158 .add(predOps(ARMCC::AL));2159 }2160 MachineInstrBuilder PopMIB2 =2161 BuildMI(MBB, MBBI, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL));2162 for (int R = 0; R < 4; ++R)2163 PopMIB2.addReg(ARM::R4 + R, RegState::Define);2164 } else { // pop Lo and Hi registers with a single instruction2165 MachineInstrBuilder PopMIB =2166 BuildMI(MBB, MBBI, DL, TII.get(ARM::t2LDMIA_UPD), ARM::SP)2167 .addReg(ARM::SP)2168 .add(predOps(ARMCC::AL));2169 for (unsigned Reg = ARM::R4; Reg < ARM::R12; ++Reg)2170 PopMIB.addReg(Reg, RegState::Define);2171 }2172}2173 2174bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,2175 MachineBasicBlock::iterator MBBI,2176 MachineBasicBlock::iterator &NextMBBI) {2177 MachineInstr &MI = *MBBI;2178 unsigned Opcode = MI.getOpcode();2179 switch (Opcode) {2180 default:2181 return false;2182 2183 case ARM::VBSPd:2184 case ARM::VBSPq: {2185 Register DstReg = MI.getOperand(0).getReg();2186 if (DstReg == MI.getOperand(3).getReg()) {2187 // Expand to VBIT2188 unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBITd : ARM::VBITq;2189 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))2190 .add(MI.getOperand(0))2191 .add(MI.getOperand(3))2192 .add(MI.getOperand(2))2193 .add(MI.getOperand(1))2194 .addImm(MI.getOperand(4).getImm())2195 .add(MI.getOperand(5));2196 } else if (DstReg == MI.getOperand(2).getReg()) {2197 // Expand to VBIF2198 unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBIFd : ARM::VBIFq;2199 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))2200 .add(MI.getOperand(0))2201 .add(MI.getOperand(2))2202 .add(MI.getOperand(3))2203 .add(MI.getOperand(1))2204 .addImm(MI.getOperand(4).getImm())2205 .add(MI.getOperand(5));2206 } else {2207 // Expand to VBSL2208 unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBSLd : ARM::VBSLq;2209 if (DstReg == MI.getOperand(1).getReg()) {2210 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))2211 .add(MI.getOperand(0))2212 .add(MI.getOperand(1))2213 .add(MI.getOperand(2))2214 .add(MI.getOperand(3))2215 .addImm(MI.getOperand(4).getImm())2216 .add(MI.getOperand(5));2217 } else {2218 // Use move to satisfy constraints2219 unsigned MoveOpc = Opcode == ARM::VBSPd ? ARM::VORRd : ARM::VORRq;2220 unsigned MO1Flags = getRegState(MI.getOperand(1)) & ~RegState::Kill;2221 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MoveOpc))2222 .addReg(DstReg,2223 RegState::Define |2224 getRenamableRegState(MI.getOperand(0).isRenamable()))2225 .addReg(MI.getOperand(1).getReg(), MO1Flags)2226 .addReg(MI.getOperand(1).getReg(), MO1Flags)2227 .addImm(MI.getOperand(4).getImm())2228 .add(MI.getOperand(5));2229 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))2230 .add(MI.getOperand(0))2231 .addReg(DstReg,2232 RegState::Kill |2233 getRenamableRegState(MI.getOperand(0).isRenamable()))2234 .add(MI.getOperand(2))2235 .add(MI.getOperand(3))2236 .addImm(MI.getOperand(4).getImm())2237 .add(MI.getOperand(5));2238 }2239 }2240 MI.eraseFromParent();2241 return true;2242 }2243 2244 case ARM::TCRETURNdi:2245 case ARM::TCRETURNri:2246 case ARM::TCRETURNrinotr12: {2247 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();2248 if (MBBI->getOpcode() == ARM::SEH_EpilogEnd)2249 MBBI--;2250 if (MBBI->getOpcode() == ARM::SEH_Nop_Ret)2251 MBBI--;2252 assert(MBBI->isReturn() &&2253 "Can only insert epilog into returning blocks");2254 unsigned RetOpcode = MBBI->getOpcode();2255 DebugLoc dl = MBBI->getDebugLoc();2256 const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(2257 MBB.getParent()->getSubtarget().getInstrInfo());2258 2259 // Tail call return: adjust the stack pointer and jump to callee.2260 MBBI = MBB.getLastNonDebugInstr();2261 if (MBBI->getOpcode() == ARM::SEH_EpilogEnd)2262 MBBI--;2263 if (MBBI->getOpcode() == ARM::SEH_Nop_Ret)2264 MBBI--;2265 MachineOperand &JumpTarget = MBBI->getOperand(0);2266 2267 // Jump to label or value in register.2268 if (RetOpcode == ARM::TCRETURNdi) {2269 MachineFunction *MF = MBB.getParent();2270 bool NeedsWinCFI = MF->getTarget().getMCAsmInfo()->usesWindowsCFI() &&2271 MF->getFunction().needsUnwindTableEntry();2272 unsigned TCOpcode =2273 STI->isThumb()2274 ? ((STI->isTargetMachO() || NeedsWinCFI) ? ARM::tTAILJMPd2275 : ARM::tTAILJMPdND)2276 : ARM::TAILJMPd;2277 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));2278 if (JumpTarget.isGlobal())2279 MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),2280 JumpTarget.getTargetFlags());2281 else {2282 assert(JumpTarget.isSymbol());2283 MIB.addExternalSymbol(JumpTarget.getSymbolName(),2284 JumpTarget.getTargetFlags());2285 }2286 2287 // Add the default predicate in Thumb mode.2288 if (STI->isThumb())2289 MIB.add(predOps(ARMCC::AL));2290 } else if (RetOpcode == ARM::TCRETURNri ||2291 RetOpcode == ARM::TCRETURNrinotr12) {2292 unsigned Opcode =2293 STI->isThumb() ? ARM::tTAILJMPr2294 : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);2295 BuildMI(MBB, MBBI, dl,2296 TII.get(Opcode))2297 .addReg(JumpTarget.getReg(), RegState::Kill);2298 }2299 2300 auto NewMI = std::prev(MBBI);2301 for (unsigned i = 2, e = MBBI->getNumOperands(); i != e; ++i)2302 NewMI->addOperand(MBBI->getOperand(i));2303 2304 NewMI->setCFIType(*MBB.getParent(), MI.getCFIType());2305 2306 // Update call info and delete the pseudo instruction TCRETURN.2307 if (MI.isCandidateForAdditionalCallInfo())2308 MI.getMF()->moveAdditionalCallInfo(&MI, &*NewMI);2309 // Copy nomerge flag over to new instruction.2310 if (MI.getFlag(MachineInstr::NoMerge))2311 NewMI->setFlag(MachineInstr::NoMerge);2312 MBB.erase(MBBI);2313 2314 MBBI = NewMI;2315 return true;2316 }2317 case ARM::tBXNS_RET: {2318 // For v8.0-M.Main we need to authenticate LR before clearing FPRs, which2319 // uses R12 as a scratch register.2320 if (!STI->hasV8_1MMainlineOps() && AFI->shouldSignReturnAddress())2321 BuildMI(MBB, MBBI, DebugLoc(), TII->get(ARM::t2AUT));2322 2323 MachineBasicBlock &AfterBB = CMSEClearFPRegs(MBB, MBBI);2324 2325 if (STI->hasV8_1MMainlineOps()) {2326 // Restore the non-secure floating point context.2327 BuildMI(MBB, MBBI, MBBI->getDebugLoc(),2328 TII->get(ARM::VLDR_FPCXTNS_post), ARM::SP)2329 .addReg(ARM::SP)2330 .addImm(4)2331 .add(predOps(ARMCC::AL));2332 2333 if (AFI->shouldSignReturnAddress())2334 BuildMI(AfterBB, AfterBB.end(), DebugLoc(), TII->get(ARM::t2AUT));2335 }2336 2337 // Clear all GPR that are not a use of the return instruction.2338 assert(llvm::all_of(MBBI->operands(), [](const MachineOperand &Op) {2339 return !Op.isReg() || Op.getReg() != ARM::R12;2340 }));2341 SmallVector<unsigned, 5> ClearRegs;2342 determineGPRegsToClear(2343 *MBBI, {ARM::R0, ARM::R1, ARM::R2, ARM::R3, ARM::R12}, ClearRegs);2344 CMSEClearGPRegs(AfterBB, AfterBB.end(), MBBI->getDebugLoc(), ClearRegs,2345 ARM::LR);2346 2347 MachineInstrBuilder NewMI =2348 BuildMI(AfterBB, AfterBB.end(), MBBI->getDebugLoc(),2349 TII->get(ARM::tBXNS))2350 .addReg(ARM::LR)2351 .add(predOps(ARMCC::AL));2352 for (const MachineOperand &Op : MI.operands())2353 NewMI->addOperand(Op);2354 MI.eraseFromParent();2355 return true;2356 }2357 case ARM::tBLXNS_CALL: {2358 DebugLoc DL = MBBI->getDebugLoc();2359 Register JumpReg = MBBI->getOperand(0).getReg();2360 2361 // Figure out which registers are live at the point immediately before the2362 // call. When we indiscriminately push a set of registers, the live2363 // registers are added as ordinary use operands, whereas dead registers2364 // are "undef".2365 LivePhysRegs LiveRegs(*TRI);2366 LiveRegs.addLiveOuts(MBB);2367 for (const MachineInstr &MI : make_range(MBB.rbegin(), MBBI.getReverse()))2368 LiveRegs.stepBackward(MI);2369 LiveRegs.stepBackward(*MBBI);2370 2371 CMSEPushCalleeSaves(*TII, MBB, MBBI, JumpReg, LiveRegs,2372 AFI->isThumb1OnlyFunction());2373 2374 SmallVector<unsigned, 16> ClearRegs;2375 determineGPRegsToClear(*MBBI,2376 {ARM::R0, ARM::R1, ARM::R2, ARM::R3, ARM::R4,2377 ARM::R5, ARM::R6, ARM::R7, ARM::R8, ARM::R9,2378 ARM::R10, ARM::R11, ARM::R12},2379 ClearRegs);2380 auto OriginalClearRegs = ClearRegs;2381 2382 // Get the first cleared register as a scratch (to use later with tBIC).2383 // We need to use the first so we can ensure it is a low register.2384 unsigned ScratchReg = ClearRegs.front();2385 2386 // Clear LSB of JumpReg2387 if (AFI->isThumb2Function()) {2388 BuildMI(MBB, MBBI, DL, TII->get(ARM::t2BICri), JumpReg)2389 .addReg(JumpReg)2390 .addImm(1)2391 .add(predOps(ARMCC::AL))2392 .add(condCodeOp());2393 } else {2394 // We need to use an extra register to cope with 8M Baseline,2395 // since we have saved all of the registers we are ok to trash a non2396 // argument register here.2397 BuildMI(MBB, MBBI, DL, TII->get(ARM::tMOVi8), ScratchReg)2398 .add(condCodeOp())2399 .addImm(1)2400 .add(predOps(ARMCC::AL));2401 BuildMI(MBB, MBBI, DL, TII->get(ARM::tBIC), JumpReg)2402 .addReg(ARM::CPSR, RegState::Define)2403 .addReg(JumpReg)2404 .addReg(ScratchReg)2405 .add(predOps(ARMCC::AL));2406 }2407 2408 CMSESaveClearFPRegs(MBB, MBBI, DL, LiveRegs,2409 ClearRegs); // save+clear FP regs with ClearRegs2410 CMSEClearGPRegs(MBB, MBBI, DL, ClearRegs, JumpReg);2411 2412 const MachineInstrBuilder NewCall =2413 BuildMI(MBB, MBBI, DL, TII->get(ARM::tBLXNSr))2414 .add(predOps(ARMCC::AL))2415 .addReg(JumpReg, RegState::Kill);2416 2417 for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))2418 NewCall->addOperand(MO);2419 if (MI.isCandidateForAdditionalCallInfo())2420 MI.getMF()->moveAdditionalCallInfo(&MI, NewCall.getInstr());2421 2422 CMSERestoreFPRegs(MBB, MBBI, DL, OriginalClearRegs); // restore FP registers2423 2424 CMSEPopCalleeSaves(*TII, MBB, MBBI, AFI->isThumb1OnlyFunction());2425 2426 MI.eraseFromParent();2427 return true;2428 }2429 case ARM::VMOVHcc:2430 case ARM::VMOVScc:2431 case ARM::VMOVDcc: {2432 unsigned newOpc = Opcode != ARM::VMOVDcc ? ARM::VMOVS : ARM::VMOVD;2433 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(newOpc),2434 MI.getOperand(1).getReg())2435 .add(MI.getOperand(2))2436 .addImm(MI.getOperand(3).getImm()) // 'pred'2437 .add(MI.getOperand(4))2438 .add(makeImplicit(MI.getOperand(1)));2439 2440 MI.eraseFromParent();2441 return true;2442 }2443 case ARM::t2MOVCCr:2444 case ARM::MOVCCr: {2445 unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVr : ARM::MOVr;2446 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),2447 MI.getOperand(1).getReg())2448 .add(MI.getOperand(2))2449 .addImm(MI.getOperand(3).getImm()) // 'pred'2450 .add(MI.getOperand(4))2451 .add(condCodeOp()) // 's' bit2452 .add(makeImplicit(MI.getOperand(1)));2453 2454 MI.eraseFromParent();2455 return true;2456 }2457 case ARM::MOVCCsi: {2458 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),2459 (MI.getOperand(1).getReg()))2460 .add(MI.getOperand(2))2461 .addImm(MI.getOperand(3).getImm())2462 .addImm(MI.getOperand(4).getImm()) // 'pred'2463 .add(MI.getOperand(5))2464 .add(condCodeOp()) // 's' bit2465 .add(makeImplicit(MI.getOperand(1)));2466 2467 MI.eraseFromParent();2468 return true;2469 }2470 case ARM::MOVCCsr: {2471 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),2472 (MI.getOperand(1).getReg()))2473 .add(MI.getOperand(2))2474 .add(MI.getOperand(3))2475 .addImm(MI.getOperand(4).getImm())2476 .addImm(MI.getOperand(5).getImm()) // 'pred'2477 .add(MI.getOperand(6))2478 .add(condCodeOp()) // 's' bit2479 .add(makeImplicit(MI.getOperand(1)));2480 2481 MI.eraseFromParent();2482 return true;2483 }2484 case ARM::t2MOVCCi16:2485 case ARM::MOVCCi16: {2486 unsigned NewOpc = AFI->isThumbFunction() ? ARM::t2MOVi16 : ARM::MOVi16;2487 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),2488 MI.getOperand(1).getReg())2489 .addImm(MI.getOperand(2).getImm())2490 .addImm(MI.getOperand(3).getImm()) // 'pred'2491 .add(MI.getOperand(4))2492 .add(makeImplicit(MI.getOperand(1)));2493 MI.eraseFromParent();2494 return true;2495 }2496 case ARM::t2MOVCCi:2497 case ARM::MOVCCi: {2498 unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVi : ARM::MOVi;2499 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),2500 MI.getOperand(1).getReg())2501 .addImm(MI.getOperand(2).getImm())2502 .addImm(MI.getOperand(3).getImm()) // 'pred'2503 .add(MI.getOperand(4))2504 .add(condCodeOp()) // 's' bit2505 .add(makeImplicit(MI.getOperand(1)));2506 2507 MI.eraseFromParent();2508 return true;2509 }2510 case ARM::t2MVNCCi:2511 case ARM::MVNCCi: {2512 unsigned Opc = AFI->isThumbFunction() ? ARM::t2MVNi : ARM::MVNi;2513 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),2514 MI.getOperand(1).getReg())2515 .addImm(MI.getOperand(2).getImm())2516 .addImm(MI.getOperand(3).getImm()) // 'pred'2517 .add(MI.getOperand(4))2518 .add(condCodeOp()) // 's' bit2519 .add(makeImplicit(MI.getOperand(1)));2520 2521 MI.eraseFromParent();2522 return true;2523 }2524 case ARM::t2MOVCClsl:2525 case ARM::t2MOVCClsr:2526 case ARM::t2MOVCCasr:2527 case ARM::t2MOVCCror: {2528 unsigned NewOpc;2529 switch (Opcode) {2530 case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break;2531 case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break;2532 case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break;2533 case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break;2534 default: llvm_unreachable("unexpeced conditional move");2535 }2536 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),2537 MI.getOperand(1).getReg())2538 .add(MI.getOperand(2))2539 .addImm(MI.getOperand(3).getImm())2540 .addImm(MI.getOperand(4).getImm()) // 'pred'2541 .add(MI.getOperand(5))2542 .add(condCodeOp()) // 's' bit2543 .add(makeImplicit(MI.getOperand(1)));2544 MI.eraseFromParent();2545 return true;2546 }2547 case ARM::Int_eh_sjlj_dispatchsetup: {2548 MachineFunction &MF = *MI.getParent()->getParent();2549 const ARMBaseRegisterInfo &RI = TII->getRegisterInfo();2550 // For functions using a base pointer, we rematerialize it (via the frame2551 // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it2552 // for us. Otherwise, expand to nothing.2553 if (RI.hasBasePointer(MF)) {2554 int32_t NumBytes = AFI->getFramePtrSpillOffset();2555 Register FramePtr = RI.getFrameRegister(MF);2556 assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&2557 "base pointer without frame pointer?");2558 2559 if (AFI->isThumb2Function()) {2560 emitT2RegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,2561 FramePtr, -NumBytes, ARMCC::AL, 0, *TII);2562 } else if (AFI->isThumbFunction()) {2563 emitThumbRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,2564 FramePtr, -NumBytes, *TII, RI);2565 } else {2566 emitARMRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,2567 FramePtr, -NumBytes, ARMCC::AL, 0,2568 *TII);2569 }2570 // If there's dynamic realignment, adjust for it.2571 if (RI.hasStackRealignment(MF)) {2572 MachineFrameInfo &MFI = MF.getFrameInfo();2573 Align MaxAlign = MFI.getMaxAlign();2574 assert (!AFI->isThumb1OnlyFunction());2575 // Emit bic r6, r6, MaxAlign2576 assert(MaxAlign <= Align(256) &&2577 "The BIC instruction cannot encode "2578 "immediates larger than 256 with all lower "2579 "bits set.");2580 unsigned bicOpc = AFI->isThumbFunction() ?2581 ARM::t2BICri : ARM::BICri;2582 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(bicOpc), ARM::R6)2583 .addReg(ARM::R6, RegState::Kill)2584 .addImm(MaxAlign.value() - 1)2585 .add(predOps(ARMCC::AL))2586 .add(condCodeOp());2587 }2588 }2589 MI.eraseFromParent();2590 return true;2591 }2592 2593 case ARM::LSRs1:2594 case ARM::ASRs1: {2595 // These are just fancy MOVs instructions.2596 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),2597 MI.getOperand(0).getReg())2598 .add(MI.getOperand(1))2599 .addImm(ARM_AM::getSORegOpc(2600 (Opcode == ARM::LSRs1 ? ARM_AM::lsr : ARM_AM::asr), 1))2601 .add(predOps(ARMCC::AL))2602 .addReg(ARM::CPSR, RegState::Define);2603 MI.eraseFromParent();2604 return true;2605 }2606 case ARM::RRX: {2607 // This encodes as "MOVs Rd, Rm, rrx2608 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),2609 MI.getOperand(0).getReg())2610 .add(MI.getOperand(1))2611 .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0))2612 .add(predOps(ARMCC::AL))2613 .add(condCodeOp())2614 .copyImplicitOps(MI);2615 MI.eraseFromParent();2616 return true;2617 }2618 case ARM::tTPsoft:2619 case ARM::TPsoft: {2620 const bool Thumb = Opcode == ARM::tTPsoft;2621 2622 MachineInstrBuilder MIB;2623 MachineFunction *MF = MBB.getParent();2624 if (STI->genLongCalls()) {2625 MachineConstantPool *MCP = MF->getConstantPool();2626 unsigned PCLabelID = AFI->createPICLabelUId();2627 MachineConstantPoolValue *CPV =2628 ARMConstantPoolSymbol::Create(MF->getFunction().getContext(),2629 "__aeabi_read_tp", PCLabelID, 0);2630 Register Reg = MI.getOperand(0).getReg();2631 MIB =2632 BuildMI(MBB, MBBI, MI.getDebugLoc(),2633 TII->get(Thumb ? ARM::tLDRpci : ARM::LDRi12), Reg)2634 .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, Align(4)));2635 if (!Thumb)2636 MIB.addImm(0);2637 MIB.add(predOps(ARMCC::AL));2638 2639 MIB =2640 BuildMI(MBB, MBBI, MI.getDebugLoc(),2641 TII->get(Thumb ? gettBLXrOpcode(*MF) : getBLXOpcode(*MF)));2642 if (Thumb)2643 MIB.add(predOps(ARMCC::AL));2644 MIB.addReg(Reg, RegState::Kill);2645 } else {2646 MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),2647 TII->get(Thumb ? ARM::tBL : ARM::BL));2648 if (Thumb)2649 MIB.add(predOps(ARMCC::AL));2650 MIB.addExternalSymbol("__aeabi_read_tp", 0);2651 }2652 2653 MIB.cloneMemRefs(MI);2654 MIB.copyImplicitOps(MI);2655 // Update the call info.2656 if (MI.isCandidateForAdditionalCallInfo())2657 MF->moveAdditionalCallInfo(&MI, &*MIB);2658 MI.eraseFromParent();2659 return true;2660 }2661 case ARM::tLDRpci_pic:2662 case ARM::t2LDRpci_pic: {2663 unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)2664 ? ARM::tLDRpci : ARM::t2LDRpci;2665 Register DstReg = MI.getOperand(0).getReg();2666 bool DstIsDead = MI.getOperand(0).isDead();2667 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewLdOpc), DstReg)2668 .add(MI.getOperand(1))2669 .add(predOps(ARMCC::AL))2670 .cloneMemRefs(MI)2671 .copyImplicitOps(MI);2672 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPICADD))2673 .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))2674 .addReg(DstReg)2675 .add(MI.getOperand(2))2676 .copyImplicitOps(MI);2677 MI.eraseFromParent();2678 return true;2679 }2680 2681 case ARM::LDRLIT_ga_abs:2682 case ARM::LDRLIT_ga_pcrel:2683 case ARM::LDRLIT_ga_pcrel_ldr:2684 case ARM::tLDRLIT_ga_abs:2685 case ARM::t2LDRLIT_ga_pcrel:2686 case ARM::tLDRLIT_ga_pcrel: {2687 Register DstReg = MI.getOperand(0).getReg();2688 bool DstIsDead = MI.getOperand(0).isDead();2689 const MachineOperand &MO1 = MI.getOperand(1);2690 auto Flags = MO1.getTargetFlags();2691 const GlobalValue *GV = MO1.getGlobal();2692 bool IsARM = Opcode != ARM::tLDRLIT_ga_pcrel &&2693 Opcode != ARM::tLDRLIT_ga_abs &&2694 Opcode != ARM::t2LDRLIT_ga_pcrel;2695 bool IsPIC =2696 Opcode != ARM::LDRLIT_ga_abs && Opcode != ARM::tLDRLIT_ga_abs;2697 unsigned LDRLITOpc = IsARM ? ARM::LDRi12 : ARM::tLDRpci;2698 if (Opcode == ARM::t2LDRLIT_ga_pcrel)2699 LDRLITOpc = ARM::t2LDRpci;2700 unsigned PICAddOpc =2701 IsARM2702 ? (Opcode == ARM::LDRLIT_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)2703 : ARM::tPICADD;2704 2705 // We need a new const-pool entry to load from.2706 MachineConstantPool *MCP = MBB.getParent()->getConstantPool();2707 unsigned ARMPCLabelIndex = 0;2708 MachineConstantPoolValue *CPV;2709 2710 if (IsPIC) {2711 unsigned PCAdj = IsARM ? 8 : 4;2712 auto Modifier = (Flags & ARMII::MO_GOT)2713 ? ARMCP::GOT_PREL2714 : ARMCP::no_modifier;2715 ARMPCLabelIndex = AFI->createPICLabelUId();2716 CPV = ARMConstantPoolConstant::Create(2717 GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj, Modifier,2718 /*AddCurrentAddr*/ Modifier == ARMCP::GOT_PREL);2719 } else2720 CPV = ARMConstantPoolConstant::Create(GV, ARMCP::no_modifier);2721 2722 MachineInstrBuilder MIB =2723 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LDRLITOpc), DstReg)2724 .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, Align(4)));2725 if (IsARM)2726 MIB.addImm(0);2727 MIB.add(predOps(ARMCC::AL));2728 2729 if (IsPIC) {2730 MachineInstrBuilder MIB =2731 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(PICAddOpc))2732 .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))2733 .addReg(DstReg)2734 .addImm(ARMPCLabelIndex);2735 2736 if (IsARM)2737 MIB.add(predOps(ARMCC::AL));2738 }2739 2740 MI.eraseFromParent();2741 return true;2742 }2743 case ARM::MOV_ga_pcrel:2744 case ARM::MOV_ga_pcrel_ldr:2745 case ARM::t2MOV_ga_pcrel: {2746 // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.2747 unsigned LabelId = AFI->createPICLabelUId();2748 Register DstReg = MI.getOperand(0).getReg();2749 bool DstIsDead = MI.getOperand(0).isDead();2750 const MachineOperand &MO1 = MI.getOperand(1);2751 const GlobalValue *GV = MO1.getGlobal();2752 unsigned TF = MO1.getTargetFlags();2753 bool isARM = Opcode != ARM::t2MOV_ga_pcrel;2754 unsigned LO16Opc = isARM ? ARM::MOVi16_ga_pcrel : ARM::t2MOVi16_ga_pcrel;2755 unsigned HI16Opc = isARM ? ARM::MOVTi16_ga_pcrel :ARM::t2MOVTi16_ga_pcrel;2756 unsigned LO16TF = TF | ARMII::MO_LO16;2757 unsigned HI16TF = TF | ARMII::MO_HI16;2758 unsigned PICAddOpc = isARM2759 ? (Opcode == ARM::MOV_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)2760 : ARM::tPICADD;2761 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg)2762 .addGlobalAddress(GV, MO1.getOffset(), TF | LO16TF)2763 .addImm(LabelId)2764 .copyImplicitOps(MI);2765 2766 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc), DstReg)2767 .addReg(DstReg)2768 .addGlobalAddress(GV, MO1.getOffset(), TF | HI16TF)2769 .addImm(LabelId)2770 .copyImplicitOps(MI);2771 2772 MachineInstrBuilder MIB3 = BuildMI(MBB, MBBI, MI.getDebugLoc(),2773 TII->get(PICAddOpc))2774 .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))2775 .addReg(DstReg).addImm(LabelId);2776 if (isARM) {2777 MIB3.add(predOps(ARMCC::AL));2778 if (Opcode == ARM::MOV_ga_pcrel_ldr)2779 MIB3.cloneMemRefs(MI);2780 }2781 MIB3.copyImplicitOps(MI);2782 MI.eraseFromParent();2783 return true;2784 }2785 2786 case ARM::MOVi32imm:2787 case ARM::MOVCCi32imm:2788 case ARM::t2MOVi32imm:2789 case ARM::t2MOVCCi32imm:2790 ExpandMOV32BitImm(MBB, MBBI);2791 return true;2792 2793 case ARM::tMOVi32imm:2794 ExpandTMOV32BitImm(MBB, MBBI);2795 return true;2796 2797 case ARM::tLEApcrelJT:2798 // Inline jump tables are handled in ARMAsmPrinter.2799 if (MI.getMF()->getJumpTableInfo()->getEntryKind() ==2800 MachineJumpTableInfo::EK_Inline)2801 return false;2802 2803 // Use a 32-bit immediate move to generate the address of the jump table.2804 assert(STI->isThumb() && "Non-inline jump tables expected only in thumb");2805 ExpandTMOV32BitImm(MBB, MBBI);2806 return true;2807 2808 case ARM::SUBS_PC_LR: {2809 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri), ARM::PC)2810 .addReg(ARM::LR)2811 .add(MI.getOperand(0))2812 .add(MI.getOperand(1))2813 .add(MI.getOperand(2))2814 .addReg(ARM::CPSR, RegState::Undef)2815 .copyImplicitOps(MI);2816 MI.eraseFromParent();2817 return true;2818 }2819 case ARM::VLDMQIA: {2820 unsigned NewOpc = ARM::VLDMDIA;2821 MachineInstrBuilder MIB =2822 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));2823 unsigned OpIdx = 0;2824 2825 // Grab the Q register destination.2826 bool DstIsDead = MI.getOperand(OpIdx).isDead();2827 Register DstReg = MI.getOperand(OpIdx++).getReg();2828 2829 // Copy the source register.2830 MIB.add(MI.getOperand(OpIdx++));2831 2832 // Copy the predicate operands.2833 MIB.add(MI.getOperand(OpIdx++));2834 MIB.add(MI.getOperand(OpIdx++));2835 2836 // Add the destination operands (D subregs).2837 Register D0 = TRI->getSubReg(DstReg, ARM::dsub_0);2838 Register D1 = TRI->getSubReg(DstReg, ARM::dsub_1);2839 MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead))2840 .addReg(D1, RegState::Define | getDeadRegState(DstIsDead));2841 2842 // Add an implicit def for the super-register.2843 MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));2844 MIB.copyImplicitOps(MI);2845 MIB.cloneMemRefs(MI);2846 MI.eraseFromParent();2847 return true;2848 }2849 2850 case ARM::VSTMQIA: {2851 unsigned NewOpc = ARM::VSTMDIA;2852 MachineInstrBuilder MIB =2853 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));2854 unsigned OpIdx = 0;2855 2856 // Grab the Q register source.2857 bool SrcIsKill = MI.getOperand(OpIdx).isKill();2858 Register SrcReg = MI.getOperand(OpIdx++).getReg();2859 2860 // Copy the destination register.2861 MachineOperand Dst(MI.getOperand(OpIdx++));2862 MIB.add(Dst);2863 2864 // Copy the predicate operands.2865 MIB.add(MI.getOperand(OpIdx++));2866 MIB.add(MI.getOperand(OpIdx++));2867 2868 // Add the source operands (D subregs).2869 Register D0 = TRI->getSubReg(SrcReg, ARM::dsub_0);2870 Register D1 = TRI->getSubReg(SrcReg, ARM::dsub_1);2871 MIB.addReg(D0, SrcIsKill ? RegState::Kill : 0)2872 .addReg(D1, SrcIsKill ? RegState::Kill : 0);2873 2874 if (SrcIsKill) // Add an implicit kill for the Q register.2875 MIB->addRegisterKilled(SrcReg, TRI, true);2876 2877 MIB.copyImplicitOps(MI);2878 MIB.cloneMemRefs(MI);2879 MI.eraseFromParent();2880 return true;2881 }2882 2883 case ARM::VLD2q8Pseudo:2884 case ARM::VLD2q16Pseudo:2885 case ARM::VLD2q32Pseudo:2886 case ARM::VLD2q8PseudoWB_fixed:2887 case ARM::VLD2q16PseudoWB_fixed:2888 case ARM::VLD2q32PseudoWB_fixed:2889 case ARM::VLD2q8PseudoWB_register:2890 case ARM::VLD2q16PseudoWB_register:2891 case ARM::VLD2q32PseudoWB_register:2892 case ARM::VLD3d8Pseudo:2893 case ARM::VLD3d16Pseudo:2894 case ARM::VLD3d32Pseudo:2895 case ARM::VLD1d8TPseudo:2896 case ARM::VLD1d8TPseudoWB_fixed:2897 case ARM::VLD1d8TPseudoWB_register:2898 case ARM::VLD1d16TPseudo:2899 case ARM::VLD1d16TPseudoWB_fixed:2900 case ARM::VLD1d16TPseudoWB_register:2901 case ARM::VLD1d32TPseudo:2902 case ARM::VLD1d32TPseudoWB_fixed:2903 case ARM::VLD1d32TPseudoWB_register:2904 case ARM::VLD1d64TPseudo:2905 case ARM::VLD1d64TPseudoWB_fixed:2906 case ARM::VLD1d64TPseudoWB_register:2907 case ARM::VLD3d8Pseudo_UPD:2908 case ARM::VLD3d16Pseudo_UPD:2909 case ARM::VLD3d32Pseudo_UPD:2910 case ARM::VLD3q8Pseudo_UPD:2911 case ARM::VLD3q16Pseudo_UPD:2912 case ARM::VLD3q32Pseudo_UPD:2913 case ARM::VLD3q8oddPseudo:2914 case ARM::VLD3q16oddPseudo:2915 case ARM::VLD3q32oddPseudo:2916 case ARM::VLD3q8oddPseudo_UPD:2917 case ARM::VLD3q16oddPseudo_UPD:2918 case ARM::VLD3q32oddPseudo_UPD:2919 case ARM::VLD4d8Pseudo:2920 case ARM::VLD4d16Pseudo:2921 case ARM::VLD4d32Pseudo:2922 case ARM::VLD1d8QPseudo:2923 case ARM::VLD1d8QPseudoWB_fixed:2924 case ARM::VLD1d8QPseudoWB_register:2925 case ARM::VLD1d16QPseudo:2926 case ARM::VLD1d16QPseudoWB_fixed:2927 case ARM::VLD1d16QPseudoWB_register:2928 case ARM::VLD1d32QPseudo:2929 case ARM::VLD1d32QPseudoWB_fixed:2930 case ARM::VLD1d32QPseudoWB_register:2931 case ARM::VLD1d64QPseudo:2932 case ARM::VLD1d64QPseudoWB_fixed:2933 case ARM::VLD1d64QPseudoWB_register:2934 case ARM::VLD1q8HighQPseudo:2935 case ARM::VLD1q8HighQPseudo_UPD:2936 case ARM::VLD1q8LowQPseudo_UPD:2937 case ARM::VLD1q8HighTPseudo:2938 case ARM::VLD1q8HighTPseudo_UPD:2939 case ARM::VLD1q8LowTPseudo_UPD:2940 case ARM::VLD1q16HighQPseudo:2941 case ARM::VLD1q16HighQPseudo_UPD:2942 case ARM::VLD1q16LowQPseudo_UPD:2943 case ARM::VLD1q16HighTPseudo:2944 case ARM::VLD1q16HighTPseudo_UPD:2945 case ARM::VLD1q16LowTPseudo_UPD:2946 case ARM::VLD1q32HighQPseudo:2947 case ARM::VLD1q32HighQPseudo_UPD:2948 case ARM::VLD1q32LowQPseudo_UPD:2949 case ARM::VLD1q32HighTPseudo:2950 case ARM::VLD1q32HighTPseudo_UPD:2951 case ARM::VLD1q32LowTPseudo_UPD:2952 case ARM::VLD1q64HighQPseudo:2953 case ARM::VLD1q64HighQPseudo_UPD:2954 case ARM::VLD1q64LowQPseudo_UPD:2955 case ARM::VLD1q64HighTPseudo:2956 case ARM::VLD1q64HighTPseudo_UPD:2957 case ARM::VLD1q64LowTPseudo_UPD:2958 case ARM::VLD4d8Pseudo_UPD:2959 case ARM::VLD4d16Pseudo_UPD:2960 case ARM::VLD4d32Pseudo_UPD:2961 case ARM::VLD4q8Pseudo_UPD:2962 case ARM::VLD4q16Pseudo_UPD:2963 case ARM::VLD4q32Pseudo_UPD:2964 case ARM::VLD4q8oddPseudo:2965 case ARM::VLD4q16oddPseudo:2966 case ARM::VLD4q32oddPseudo:2967 case ARM::VLD4q8oddPseudo_UPD:2968 case ARM::VLD4q16oddPseudo_UPD:2969 case ARM::VLD4q32oddPseudo_UPD:2970 case ARM::VLD3DUPd8Pseudo:2971 case ARM::VLD3DUPd16Pseudo:2972 case ARM::VLD3DUPd32Pseudo:2973 case ARM::VLD3DUPd8Pseudo_UPD:2974 case ARM::VLD3DUPd16Pseudo_UPD:2975 case ARM::VLD3DUPd32Pseudo_UPD:2976 case ARM::VLD4DUPd8Pseudo:2977 case ARM::VLD4DUPd16Pseudo:2978 case ARM::VLD4DUPd32Pseudo:2979 case ARM::VLD4DUPd8Pseudo_UPD:2980 case ARM::VLD4DUPd16Pseudo_UPD:2981 case ARM::VLD4DUPd32Pseudo_UPD:2982 case ARM::VLD2DUPq8EvenPseudo:2983 case ARM::VLD2DUPq8OddPseudo:2984 case ARM::VLD2DUPq16EvenPseudo:2985 case ARM::VLD2DUPq16OddPseudo:2986 case ARM::VLD2DUPq32EvenPseudo:2987 case ARM::VLD2DUPq32OddPseudo:2988 case ARM::VLD2DUPq8OddPseudoWB_fixed:2989 case ARM::VLD2DUPq8OddPseudoWB_register:2990 case ARM::VLD2DUPq16OddPseudoWB_fixed:2991 case ARM::VLD2DUPq16OddPseudoWB_register:2992 case ARM::VLD2DUPq32OddPseudoWB_fixed:2993 case ARM::VLD2DUPq32OddPseudoWB_register:2994 case ARM::VLD3DUPq8EvenPseudo:2995 case ARM::VLD3DUPq8OddPseudo:2996 case ARM::VLD3DUPq16EvenPseudo:2997 case ARM::VLD3DUPq16OddPseudo:2998 case ARM::VLD3DUPq32EvenPseudo:2999 case ARM::VLD3DUPq32OddPseudo:3000 case ARM::VLD3DUPq8OddPseudo_UPD:3001 case ARM::VLD3DUPq16OddPseudo_UPD:3002 case ARM::VLD3DUPq32OddPseudo_UPD:3003 case ARM::VLD4DUPq8EvenPseudo:3004 case ARM::VLD4DUPq8OddPseudo:3005 case ARM::VLD4DUPq16EvenPseudo:3006 case ARM::VLD4DUPq16OddPseudo:3007 case ARM::VLD4DUPq32EvenPseudo:3008 case ARM::VLD4DUPq32OddPseudo:3009 case ARM::VLD4DUPq8OddPseudo_UPD:3010 case ARM::VLD4DUPq16OddPseudo_UPD:3011 case ARM::VLD4DUPq32OddPseudo_UPD:3012 ExpandVLD(MBBI);3013 return true;3014 3015 case ARM::VST2q8Pseudo:3016 case ARM::VST2q16Pseudo:3017 case ARM::VST2q32Pseudo:3018 case ARM::VST2q8PseudoWB_fixed:3019 case ARM::VST2q16PseudoWB_fixed:3020 case ARM::VST2q32PseudoWB_fixed:3021 case ARM::VST2q8PseudoWB_register:3022 case ARM::VST2q16PseudoWB_register:3023 case ARM::VST2q32PseudoWB_register:3024 case ARM::VST3d8Pseudo:3025 case ARM::VST3d16Pseudo:3026 case ARM::VST3d32Pseudo:3027 case ARM::VST1d8TPseudo:3028 case ARM::VST1d8TPseudoWB_fixed:3029 case ARM::VST1d8TPseudoWB_register:3030 case ARM::VST1d16TPseudo:3031 case ARM::VST1d16TPseudoWB_fixed:3032 case ARM::VST1d16TPseudoWB_register:3033 case ARM::VST1d32TPseudo:3034 case ARM::VST1d32TPseudoWB_fixed:3035 case ARM::VST1d32TPseudoWB_register:3036 case ARM::VST1d64TPseudo:3037 case ARM::VST1d64TPseudoWB_fixed:3038 case ARM::VST1d64TPseudoWB_register:3039 case ARM::VST3d8Pseudo_UPD:3040 case ARM::VST3d16Pseudo_UPD:3041 case ARM::VST3d32Pseudo_UPD:3042 case ARM::VST3q8Pseudo_UPD:3043 case ARM::VST3q16Pseudo_UPD:3044 case ARM::VST3q32Pseudo_UPD:3045 case ARM::VST3q8oddPseudo:3046 case ARM::VST3q16oddPseudo:3047 case ARM::VST3q32oddPseudo:3048 case ARM::VST3q8oddPseudo_UPD:3049 case ARM::VST3q16oddPseudo_UPD:3050 case ARM::VST3q32oddPseudo_UPD:3051 case ARM::VST4d8Pseudo:3052 case ARM::VST4d16Pseudo:3053 case ARM::VST4d32Pseudo:3054 case ARM::VST1d8QPseudo:3055 case ARM::VST1d8QPseudoWB_fixed:3056 case ARM::VST1d8QPseudoWB_register:3057 case ARM::VST1d16QPseudo:3058 case ARM::VST1d16QPseudoWB_fixed:3059 case ARM::VST1d16QPseudoWB_register:3060 case ARM::VST1d32QPseudo:3061 case ARM::VST1d32QPseudoWB_fixed:3062 case ARM::VST1d32QPseudoWB_register:3063 case ARM::VST1d64QPseudo:3064 case ARM::VST1d64QPseudoWB_fixed:3065 case ARM::VST1d64QPseudoWB_register:3066 case ARM::VST4d8Pseudo_UPD:3067 case ARM::VST4d16Pseudo_UPD:3068 case ARM::VST4d32Pseudo_UPD:3069 case ARM::VST1q8HighQPseudo:3070 case ARM::VST1q8LowQPseudo_UPD:3071 case ARM::VST1q8HighTPseudo:3072 case ARM::VST1q8LowTPseudo_UPD:3073 case ARM::VST1q16HighQPseudo:3074 case ARM::VST1q16LowQPseudo_UPD:3075 case ARM::VST1q16HighTPseudo:3076 case ARM::VST1q16LowTPseudo_UPD:3077 case ARM::VST1q32HighQPseudo:3078 case ARM::VST1q32LowQPseudo_UPD:3079 case ARM::VST1q32HighTPseudo:3080 case ARM::VST1q32LowTPseudo_UPD:3081 case ARM::VST1q64HighQPseudo:3082 case ARM::VST1q64LowQPseudo_UPD:3083 case ARM::VST1q64HighTPseudo:3084 case ARM::VST1q64LowTPseudo_UPD:3085 case ARM::VST1q8HighTPseudo_UPD:3086 case ARM::VST1q16HighTPseudo_UPD:3087 case ARM::VST1q32HighTPseudo_UPD:3088 case ARM::VST1q64HighTPseudo_UPD:3089 case ARM::VST1q8HighQPseudo_UPD:3090 case ARM::VST1q16HighQPseudo_UPD:3091 case ARM::VST1q32HighQPseudo_UPD:3092 case ARM::VST1q64HighQPseudo_UPD:3093 case ARM::VST4q8Pseudo_UPD:3094 case ARM::VST4q16Pseudo_UPD:3095 case ARM::VST4q32Pseudo_UPD:3096 case ARM::VST4q8oddPseudo:3097 case ARM::VST4q16oddPseudo:3098 case ARM::VST4q32oddPseudo:3099 case ARM::VST4q8oddPseudo_UPD:3100 case ARM::VST4q16oddPseudo_UPD:3101 case ARM::VST4q32oddPseudo_UPD:3102 ExpandVST(MBBI);3103 return true;3104 3105 case ARM::VLD1LNq8Pseudo:3106 case ARM::VLD1LNq16Pseudo:3107 case ARM::VLD1LNq32Pseudo:3108 case ARM::VLD1LNq8Pseudo_UPD:3109 case ARM::VLD1LNq16Pseudo_UPD:3110 case ARM::VLD1LNq32Pseudo_UPD:3111 case ARM::VLD2LNd8Pseudo:3112 case ARM::VLD2LNd16Pseudo:3113 case ARM::VLD2LNd32Pseudo:3114 case ARM::VLD2LNq16Pseudo:3115 case ARM::VLD2LNq32Pseudo:3116 case ARM::VLD2LNd8Pseudo_UPD:3117 case ARM::VLD2LNd16Pseudo_UPD:3118 case ARM::VLD2LNd32Pseudo_UPD:3119 case ARM::VLD2LNq16Pseudo_UPD:3120 case ARM::VLD2LNq32Pseudo_UPD:3121 case ARM::VLD3LNd8Pseudo:3122 case ARM::VLD3LNd16Pseudo:3123 case ARM::VLD3LNd32Pseudo:3124 case ARM::VLD3LNq16Pseudo:3125 case ARM::VLD3LNq32Pseudo:3126 case ARM::VLD3LNd8Pseudo_UPD:3127 case ARM::VLD3LNd16Pseudo_UPD:3128 case ARM::VLD3LNd32Pseudo_UPD:3129 case ARM::VLD3LNq16Pseudo_UPD:3130 case ARM::VLD3LNq32Pseudo_UPD:3131 case ARM::VLD4LNd8Pseudo:3132 case ARM::VLD4LNd16Pseudo:3133 case ARM::VLD4LNd32Pseudo:3134 case ARM::VLD4LNq16Pseudo:3135 case ARM::VLD4LNq32Pseudo:3136 case ARM::VLD4LNd8Pseudo_UPD:3137 case ARM::VLD4LNd16Pseudo_UPD:3138 case ARM::VLD4LNd32Pseudo_UPD:3139 case ARM::VLD4LNq16Pseudo_UPD:3140 case ARM::VLD4LNq32Pseudo_UPD:3141 case ARM::VST1LNq8Pseudo:3142 case ARM::VST1LNq16Pseudo:3143 case ARM::VST1LNq32Pseudo:3144 case ARM::VST1LNq8Pseudo_UPD:3145 case ARM::VST1LNq16Pseudo_UPD:3146 case ARM::VST1LNq32Pseudo_UPD:3147 case ARM::VST2LNd8Pseudo:3148 case ARM::VST2LNd16Pseudo:3149 case ARM::VST2LNd32Pseudo:3150 case ARM::VST2LNq16Pseudo:3151 case ARM::VST2LNq32Pseudo:3152 case ARM::VST2LNd8Pseudo_UPD:3153 case ARM::VST2LNd16Pseudo_UPD:3154 case ARM::VST2LNd32Pseudo_UPD:3155 case ARM::VST2LNq16Pseudo_UPD:3156 case ARM::VST2LNq32Pseudo_UPD:3157 case ARM::VST3LNd8Pseudo:3158 case ARM::VST3LNd16Pseudo:3159 case ARM::VST3LNd32Pseudo:3160 case ARM::VST3LNq16Pseudo:3161 case ARM::VST3LNq32Pseudo:3162 case ARM::VST3LNd8Pseudo_UPD:3163 case ARM::VST3LNd16Pseudo_UPD:3164 case ARM::VST3LNd32Pseudo_UPD:3165 case ARM::VST3LNq16Pseudo_UPD:3166 case ARM::VST3LNq32Pseudo_UPD:3167 case ARM::VST4LNd8Pseudo:3168 case ARM::VST4LNd16Pseudo:3169 case ARM::VST4LNd32Pseudo:3170 case ARM::VST4LNq16Pseudo:3171 case ARM::VST4LNq32Pseudo:3172 case ARM::VST4LNd8Pseudo_UPD:3173 case ARM::VST4LNd16Pseudo_UPD:3174 case ARM::VST4LNd32Pseudo_UPD:3175 case ARM::VST4LNq16Pseudo_UPD:3176 case ARM::VST4LNq32Pseudo_UPD:3177 ExpandLaneOp(MBBI);3178 return true;3179 3180 case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, ARM::VTBL3, false); return true;3181 case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true;3182 case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true;3183 case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true;3184 3185 case ARM::MQQPRLoad:3186 case ARM::MQQPRStore:3187 case ARM::MQQQQPRLoad:3188 case ARM::MQQQQPRStore:3189 ExpandMQQPRLoadStore(MBBI);3190 return true;3191 3192 case ARM::tCMP_SWAP_8:3193 assert(STI->isThumb());3194 return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB, ARM::tUXTB,3195 NextMBBI);3196 case ARM::tCMP_SWAP_16:3197 assert(STI->isThumb());3198 return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH, ARM::tUXTH,3199 NextMBBI);3200 case ARM::tCMP_SWAP_32:3201 assert(STI->isThumb());3202 return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREX, ARM::t2STREX, 0, NextMBBI);3203 3204 case ARM::CMP_SWAP_8:3205 assert(!STI->isThumb());3206 return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB, ARM::UXTB,3207 NextMBBI);3208 case ARM::CMP_SWAP_16:3209 assert(!STI->isThumb());3210 return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH, ARM::UXTH,3211 NextMBBI);3212 case ARM::CMP_SWAP_32:3213 assert(!STI->isThumb());3214 return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREX, ARM::STREX, 0, NextMBBI);3215 3216 case ARM::CMP_SWAP_64:3217 return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);3218 3219 case ARM::tBL_PUSHLR:3220 case ARM::BL_PUSHLR: {3221 const bool Thumb = Opcode == ARM::tBL_PUSHLR;3222 Register Reg = MI.getOperand(0).getReg();3223 assert(Reg == ARM::LR && "expect LR register!");3224 MachineInstrBuilder MIB;3225 if (Thumb) {3226 // push {lr}3227 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPUSH))3228 .add(predOps(ARMCC::AL))3229 .addReg(Reg);3230 3231 // bl __gnu_mcount_nc3232 MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tBL));3233 } else {3234 // stmdb sp!, {lr}3235 BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::STMDB_UPD))3236 .addReg(ARM::SP, RegState::Define)3237 .addReg(ARM::SP)3238 .add(predOps(ARMCC::AL))3239 .addReg(Reg);3240 3241 // bl __gnu_mcount_nc3242 MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::BL));3243 }3244 MIB.cloneMemRefs(MI);3245 for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))3246 MIB.add(MO);3247 MI.eraseFromParent();3248 return true;3249 }3250 case ARM::t2CALL_BTI: {3251 MachineFunction &MF = *MI.getMF();3252 MachineInstrBuilder MIB =3253 BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::tBL));3254 MIB.cloneMemRefs(MI);3255 for (unsigned i = 0; i < MI.getNumOperands(); ++i)3256 MIB.add(MI.getOperand(i));3257 if (MI.isCandidateForAdditionalCallInfo())3258 MF.moveAdditionalCallInfo(&MI, MIB.getInstr());3259 MIBundleBuilder Bundler(MBB, MI);3260 Bundler.append(MIB);3261 Bundler.append(BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::t2BTI)));3262 finalizeBundle(MBB, Bundler.begin(), Bundler.end());3263 MI.eraseFromParent();3264 return true;3265 }3266 case ARM::LOADDUAL:3267 case ARM::STOREDUAL: {3268 Register PairReg = MI.getOperand(0).getReg();3269 3270 MachineInstrBuilder MIB =3271 BuildMI(MBB, MBBI, MI.getDebugLoc(),3272 TII->get(Opcode == ARM::LOADDUAL ? ARM::LDRD : ARM::STRD))3273 .addReg(TRI->getSubReg(PairReg, ARM::gsub_0),3274 Opcode == ARM::LOADDUAL ? RegState::Define : 0)3275 .addReg(TRI->getSubReg(PairReg, ARM::gsub_1),3276 Opcode == ARM::LOADDUAL ? RegState::Define : 0);3277 for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))3278 MIB.add(MO);3279 MIB.add(predOps(ARMCC::AL));3280 MIB.cloneMemRefs(MI);3281 MI.eraseFromParent();3282 return true;3283 }3284 }3285}3286 3287bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {3288 bool Modified = false;3289 3290 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();3291 while (MBBI != E) {3292 MachineBasicBlock::iterator NMBBI = std::next(MBBI);3293 Modified |= ExpandMI(MBB, MBBI, NMBBI);3294 MBBI = NMBBI;3295 }3296 3297 return Modified;3298}3299 3300bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {3301 STI = &MF.getSubtarget<ARMSubtarget>();3302 TII = STI->getInstrInfo();3303 TRI = STI->getRegisterInfo();3304 AFI = MF.getInfo<ARMFunctionInfo>();3305 3306 LLVM_DEBUG(dbgs() << "********** ARM EXPAND PSEUDO INSTRUCTIONS **********\n"3307 << "********** Function: " << MF.getName() << '\n');3308 3309 bool Modified = false;3310 for (MachineBasicBlock &MBB : MF)3311 Modified |= ExpandMBB(MBB);3312 if (VerifyARMPseudo)3313 MF.verify(this, "After expanding ARM pseudo instructions.");3314 3315 LLVM_DEBUG(dbgs() << "***************************************************\n");3316 return Modified;3317}3318 3319/// createARMExpandPseudoPass - returns an instance of the pseudo instruction3320/// expansion pass.3321FunctionPass *llvm::createARMExpandPseudoPass() {3322 return new ARMExpandPseudo();3323}3324