248 lines · cpp
1//===-- AArch64A53Fix835769.cpp -------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// This pass changes code to work around Cortex-A53 erratum 835769.9// It works around it by inserting a nop instruction in code sequences that10// in some circumstances may trigger the erratum.11// It inserts a nop instruction between a sequence of the following 2 classes12// of instructions:13// instr 1: mem-instr (including loads, stores and prefetches).14// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.15//===----------------------------------------------------------------------===//16 17#include "AArch64.h"18#include "AArch64Subtarget.h"19#include "llvm/ADT/Statistic.h"20#include "llvm/CodeGen/MachineFunction.h"21#include "llvm/CodeGen/MachineFunctionPass.h"22#include "llvm/CodeGen/MachineInstr.h"23#include "llvm/CodeGen/MachineInstrBuilder.h"24#include "llvm/CodeGen/TargetInstrInfo.h"25#include "llvm/Support/Debug.h"26#include "llvm/Support/raw_ostream.h"27 28using namespace llvm;29 30#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"31 32STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");33 34//===----------------------------------------------------------------------===//35// Helper functions36 37// Is the instruction a match for the instruction that comes first in the38// sequence of instructions that can trigger the erratum?39static bool isFirstInstructionInSequence(MachineInstr *MI) {40 // Must return true if this instruction is a load, a store or a prefetch.41 switch (MI->getOpcode()) {42 case AArch64::PRFMl:43 case AArch64::PRFMroW:44 case AArch64::PRFMroX:45 case AArch64::PRFMui:46 case AArch64::PRFUMi:47 return true;48 default:49 return MI->mayLoadOrStore();50 }51}52 53// Is the instruction a match for the instruction that comes second in the54// sequence that can trigger the erratum?55static bool isSecondInstructionInSequence(MachineInstr *MI) {56 // Must return true for non-SIMD integer multiply-accumulates, writing57 // to a 64-bit register.58 switch (MI->getOpcode()) {59 // Erratum cannot be triggered when the destination register is 32 bits,60 // therefore only include the following.61 case AArch64::MSUBXrrr:62 case AArch64::MADDXrrr:63 case AArch64::SMADDLrrr:64 case AArch64::SMSUBLrrr:65 case AArch64::UMADDLrrr:66 case AArch64::UMSUBLrrr:67 // Erratum can only be triggered by multiply-adds, not by regular68 // non-accumulating multiplies, i.e. when Ra=XZR='11111'69 return MI->getOperand(3).getReg() != AArch64::XZR;70 default:71 return false;72 }73}74 75 76//===----------------------------------------------------------------------===//77 78namespace {79class AArch64A53Fix835769 : public MachineFunctionPass {80 const TargetInstrInfo *TII;81 82public:83 static char ID;84 explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {}85 86 bool runOnMachineFunction(MachineFunction &F) override;87 88 MachineFunctionProperties getRequiredProperties() const override {89 return MachineFunctionProperties().setNoVRegs();90 }91 92 StringRef getPassName() const override {93 return "Workaround A53 erratum 835769 pass";94 }95 96 void getAnalysisUsage(AnalysisUsage &AU) const override {97 AU.setPreservesCFG();98 MachineFunctionPass::getAnalysisUsage(AU);99 }100 101private:102 bool runOnBasicBlock(MachineBasicBlock &MBB);103};104char AArch64A53Fix835769::ID = 0;105 106} // end anonymous namespace107 108INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",109 "AArch64 fix for A53 erratum 835769", false, false)110 111//===----------------------------------------------------------------------===//112 113bool114AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {115 LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");116 auto &STI = F.getSubtarget<AArch64Subtarget>();117 // Fix not requested, skip pass.118 if (!STI.fixCortexA53_835769())119 return false;120 121 bool Changed = false;122 TII = STI.getInstrInfo();123 124 for (auto &MBB : F) {125 Changed |= runOnBasicBlock(MBB);126 }127 return Changed;128}129 130// Return the block that was fallen through to get to MBB, if any,131// otherwise nullptr.132static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,133 const TargetInstrInfo *TII) {134 // Get the previous machine basic block in the function.135 MachineFunction::iterator MBBI(MBB);136 137 // Can't go off top of function.138 if (MBBI == MBB->getParent()->begin())139 return nullptr;140 141 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;142 SmallVector<MachineOperand, 2> Cond;143 144 MachineBasicBlock *PrevBB = &*std::prev(MBBI);145 for (MachineBasicBlock *S : MBB->predecessors())146 if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&147 !FBB)148 return S;149 150 return nullptr;151}152 153// Iterate through fallen through blocks trying to find a previous non-pseudo if154// there is one, otherwise return nullptr. Only look for instructions in155// previous blocks, not the current block, since we only use this to look at156// previous blocks.157static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,158 const TargetInstrInfo *TII) {159 MachineBasicBlock *FMBB = &MBB;160 161 // If there is no non-pseudo in the current block, loop back around and try162 // the previous block (if there is one).163 while ((FMBB = getBBFallenThrough(FMBB, TII))) {164 for (MachineInstr &I : llvm::reverse(*FMBB))165 if (!I.isPseudo())166 return &I;167 }168 169 // There was no previous non-pseudo in the fallen through blocks170 return nullptr;171}172 173static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,174 const TargetInstrInfo *TII) {175 // If we are the first instruction of the block, put the NOP at the end of176 // the previous fallthrough block177 if (MI == &MBB.front()) {178 MachineInstr *I = getLastNonPseudo(MBB, TII);179 assert(I && "Expected instruction");180 DebugLoc DL = I->getDebugLoc();181 BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);182 }183 else {184 DebugLoc DL = MI->getDebugLoc();185 BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);186 }187 188 ++NumNopsAdded;189}190 191bool192AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {193 bool Changed = false;194 LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB195 << " - scanning instructions...\n");196 197 // First, scan the basic block, looking for a sequence of 2 instructions198 // that match the conditions under which the erratum may trigger.199 200 // List of terminating instructions in matching sequences201 std::vector<MachineInstr*> Sequences;202 unsigned Idx = 0;203 MachineInstr *PrevInstr = nullptr;204 205 // Try and find the last non-pseudo instruction in any fallen through blocks,206 // if there isn't one, then we use nullptr to represent that.207 PrevInstr = getLastNonPseudo(MBB, TII);208 209 for (auto &MI : MBB) {210 MachineInstr *CurrInstr = &MI;211 LLVM_DEBUG(dbgs() << " Examining: " << MI);212 if (PrevInstr) {213 LLVM_DEBUG(dbgs() << " PrevInstr: " << *PrevInstr214 << " CurrInstr: " << *CurrInstr215 << " isFirstInstructionInSequence(PrevInstr): "216 << isFirstInstructionInSequence(PrevInstr) << "\n"217 << " isSecondInstructionInSequence(CurrInstr): "218 << isSecondInstructionInSequence(CurrInstr) << "\n");219 if (isFirstInstructionInSequence(PrevInstr) &&220 isSecondInstructionInSequence(CurrInstr)) {221 LLVM_DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n");222 (void) Idx;223 Sequences.push_back(CurrInstr);224 }225 }226 if (!CurrInstr->isPseudo())227 PrevInstr = CurrInstr;228 ++Idx;229 }230 231 LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()232 << " occurrences of pattern found.\n");233 234 // Then update the basic block, inserting nops between the detected sequences.235 for (auto &MI : Sequences) {236 Changed = true;237 insertNopBeforeInstruction(MBB, MI, TII);238 }239 240 return Changed;241}242 243// Factory function used by AArch64TargetMachine to add the pass to244// the passmanager.245FunctionPass *llvm::createAArch64A53Fix835769() {246 return new AArch64A53Fix835769();247}248