426 lines · cpp
1//===-- ARMFixCortexA57AES1742098Pass.cpp ---------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// This pass works around a Cortex Core Fused AES erratum:9// - Cortex-A57 Erratum 174209810// - Cortex-A72 Erratum 165543111//12// The erratum may be triggered if an input vector register to AESE or AESD was13// last written by an instruction that only updated 32 bits of it. This can14// occur for either of the input registers.15//16// The workaround chosen is to update the input register using `r = VORRq r, r`,17// as this updates all 128 bits of the register unconditionally, but does not18// change the values observed in `r`, making the input safe.19//20// This pass has to be conservative in a few cases:21// - an input vector register to the AES instruction is defined outside the22// current function, where we have to assume the register was updated in an23// unsafe way; and24// - an input vector register to the AES instruction is updated along multiple25// different control-flow paths, where we have to ensure all the register26// updating instructions are safe.27//28// Both of these cases may apply to a input vector register. In either case, we29// need to ensure that, when the pass is finished, there exists a safe30// instruction between every unsafe register updating instruction and the AES31// instruction.32//33//===----------------------------------------------------------------------===//34 35#include "ARM.h"36#include "ARMBaseInstrInfo.h"37#include "ARMBaseRegisterInfo.h"38#include "ARMSubtarget.h"39#include "Utils/ARMBaseInfo.h"40#include "llvm/ADT/STLExtras.h"41#include "llvm/ADT/SmallPtrSet.h"42#include "llvm/ADT/SmallVector.h"43#include "llvm/ADT/StringRef.h"44#include "llvm/CodeGen/MachineBasicBlock.h"45#include "llvm/CodeGen/MachineFunction.h"46#include "llvm/CodeGen/MachineFunctionPass.h"47#include "llvm/CodeGen/MachineInstr.h"48#include "llvm/CodeGen/MachineInstrBuilder.h"49#include "llvm/CodeGen/MachineInstrBundleIterator.h"50#include "llvm/CodeGen/MachineOperand.h"51#include "llvm/CodeGen/ReachingDefAnalysis.h"52#include "llvm/CodeGen/Register.h"53#include "llvm/CodeGen/TargetRegisterInfo.h"54#include "llvm/IR/DebugLoc.h"55#include "llvm/Pass.h"56#include "llvm/Support/Debug.h"57#include "llvm/Support/raw_ostream.h"58#include <assert.h>59#include <stdint.h>60 61using namespace llvm;62 63#define DEBUG_TYPE "arm-fix-cortex-a57-aes-1742098"64 65//===----------------------------------------------------------------------===//66 67namespace {68class ARMFixCortexA57AES1742098 : public MachineFunctionPass {69public:70 static char ID;71 explicit ARMFixCortexA57AES1742098() : MachineFunctionPass(ID) {}72 73 bool runOnMachineFunction(MachineFunction &F) override;74 75 MachineFunctionProperties getRequiredProperties() const override {76 return MachineFunctionProperties().setNoVRegs();77 }78 79 StringRef getPassName() const override {80 return "ARM fix for Cortex-A57 AES Erratum 1742098";81 }82 83 void getAnalysisUsage(AnalysisUsage &AU) const override {84 AU.addRequired<ReachingDefInfoWrapperPass>();85 AU.setPreservesCFG();86 MachineFunctionPass::getAnalysisUsage(AU);87 }88 89private:90 // This is the information needed to insert the fixup in the right place.91 struct AESFixupLocation {92 MachineBasicBlock *Block;93 // The fixup instruction will be inserted *before* InsertionPt.94 MachineInstr *InsertionPt;95 MachineOperand *MOp;96 };97 98 void analyzeMF(MachineFunction &MF, ReachingDefInfo &RDI,99 const ARMBaseRegisterInfo *TRI,100 SmallVectorImpl<AESFixupLocation> &FixupLocsForFn) const;101 102 void insertAESFixup(AESFixupLocation &FixupLoc, const ARMBaseInstrInfo *TII,103 const ARMBaseRegisterInfo *TRI) const;104 105 static bool isFirstAESPairInstr(MachineInstr &MI);106 static bool isSafeAESInput(MachineInstr &MI);107};108char ARMFixCortexA57AES1742098::ID = 0;109 110} // end anonymous namespace111 112INITIALIZE_PASS_BEGIN(ARMFixCortexA57AES1742098, DEBUG_TYPE,113 "ARM fix for Cortex-A57 AES Erratum 1742098", false,114 false)115INITIALIZE_PASS_DEPENDENCY(ReachingDefInfoWrapperPass);116INITIALIZE_PASS_END(ARMFixCortexA57AES1742098, DEBUG_TYPE,117 "ARM fix for Cortex-A57 AES Erratum 1742098", false, false)118 119//===----------------------------------------------------------------------===//120 121bool ARMFixCortexA57AES1742098::isFirstAESPairInstr(MachineInstr &MI) {122 unsigned Opc = MI.getOpcode();123 return Opc == ARM::AESD || Opc == ARM::AESE;124}125 126bool ARMFixCortexA57AES1742098::isSafeAESInput(MachineInstr &MI) {127 auto CondCodeIsAL = [](MachineInstr &MI) -> bool {128 int CCIdx = MI.findFirstPredOperandIdx();129 if (CCIdx == -1)130 return false;131 return MI.getOperand(CCIdx).getImm() == (int64_t)ARMCC::AL;132 };133 134 switch (MI.getOpcode()) {135 // Unknown: Assume not safe.136 default:137 return false;138 // 128-bit wide AES instructions139 case ARM::AESD:140 case ARM::AESE:141 case ARM::AESMC:142 case ARM::AESIMC:143 // No CondCode.144 return true;145 // 128-bit and 64-bit wide bitwise ops (when condition = al)146 case ARM::VANDd:147 case ARM::VANDq:148 case ARM::VORRd:149 case ARM::VORRq:150 case ARM::VEORd:151 case ARM::VEORq:152 case ARM::VMVNd:153 case ARM::VMVNq:154 // VMOV of 64-bit value between D registers (when condition = al)155 case ARM::VMOVD:156 // VMOV of 64 bit value from GPRs (when condition = al)157 case ARM::VMOVDRR:158 // VMOV of immediate into D or Q registers (when condition = al)159 case ARM::VMOVv2i64:160 case ARM::VMOVv1i64:161 case ARM::VMOVv2f32:162 case ARM::VMOVv4f32:163 case ARM::VMOVv2i32:164 case ARM::VMOVv4i32:165 case ARM::VMOVv4i16:166 case ARM::VMOVv8i16:167 case ARM::VMOVv8i8:168 case ARM::VMOVv16i8:169 // Loads (when condition = al)170 // VLD Dn, [Rn, #imm]171 case ARM::VLDRD:172 // VLDM173 case ARM::VLDMDDB_UPD:174 case ARM::VLDMDIA_UPD:175 case ARM::VLDMDIA:176 // VLDn to all lanes.177 case ARM::VLD1d64:178 case ARM::VLD1q64:179 case ARM::VLD1d32:180 case ARM::VLD1q32:181 case ARM::VLD2b32:182 case ARM::VLD2d32:183 case ARM::VLD2q32:184 case ARM::VLD1d16:185 case ARM::VLD1q16:186 case ARM::VLD2d16:187 case ARM::VLD2q16:188 case ARM::VLD1d8:189 case ARM::VLD1q8:190 case ARM::VLD2b8:191 case ARM::VLD2d8:192 case ARM::VLD2q8:193 case ARM::VLD3d32:194 case ARM::VLD3q32:195 case ARM::VLD3d16:196 case ARM::VLD3q16:197 case ARM::VLD3d8:198 case ARM::VLD3q8:199 case ARM::VLD4d32:200 case ARM::VLD4q32:201 case ARM::VLD4d16:202 case ARM::VLD4q16:203 case ARM::VLD4d8:204 case ARM::VLD4q8:205 // VLD1 (single element to one lane)206 case ARM::VLD1LNd32:207 case ARM::VLD1LNd32_UPD:208 case ARM::VLD1LNd8:209 case ARM::VLD1LNd8_UPD:210 case ARM::VLD1LNd16:211 case ARM::VLD1LNd16_UPD:212 // VLD1 (single element to all lanes)213 case ARM::VLD1DUPd32:214 case ARM::VLD1DUPd32wb_fixed:215 case ARM::VLD1DUPd32wb_register:216 case ARM::VLD1DUPd16:217 case ARM::VLD1DUPd16wb_fixed:218 case ARM::VLD1DUPd16wb_register:219 case ARM::VLD1DUPd8:220 case ARM::VLD1DUPd8wb_fixed:221 case ARM::VLD1DUPd8wb_register:222 case ARM::VLD1DUPq32:223 case ARM::VLD1DUPq32wb_fixed:224 case ARM::VLD1DUPq32wb_register:225 case ARM::VLD1DUPq16:226 case ARM::VLD1DUPq16wb_fixed:227 case ARM::VLD1DUPq16wb_register:228 case ARM::VLD1DUPq8:229 case ARM::VLD1DUPq8wb_fixed:230 case ARM::VLD1DUPq8wb_register:231 // VMOV232 case ARM::VSETLNi32:233 case ARM::VSETLNi16:234 case ARM::VSETLNi8:235 return CondCodeIsAL(MI);236 };237 238 return false;239}240 241bool ARMFixCortexA57AES1742098::runOnMachineFunction(MachineFunction &F) {242 LLVM_DEBUG(dbgs() << "***** ARMFixCortexA57AES1742098 *****\n");243 auto &STI = F.getSubtarget<ARMSubtarget>();244 245 // Fix not requested or AES instructions not present: skip pass.246 if (!STI.hasAES() || !STI.fixCortexA57AES1742098())247 return false;248 249 const ARMBaseRegisterInfo *TRI = STI.getRegisterInfo();250 const ARMBaseInstrInfo *TII = STI.getInstrInfo();251 252 auto &RDI = getAnalysis<ReachingDefInfoWrapperPass>().getRDI();253 254 // Analyze whole function to find instructions which need fixing up...255 SmallVector<AESFixupLocation> FixupLocsForFn{};256 analyzeMF(F, RDI, TRI, FixupLocsForFn);257 258 // ... and fix the instructions up all at the same time.259 bool Changed = false;260 LLVM_DEBUG(dbgs() << "Inserting " << FixupLocsForFn.size() << " fixup(s)\n");261 for (AESFixupLocation &FixupLoc : FixupLocsForFn) {262 insertAESFixup(FixupLoc, TII, TRI);263 Changed |= true;264 }265 266 return Changed;267}268 269void ARMFixCortexA57AES1742098::analyzeMF(270 MachineFunction &MF, ReachingDefInfo &RDI, const ARMBaseRegisterInfo *TRI,271 SmallVectorImpl<AESFixupLocation> &FixupLocsForFn) const {272 unsigned MaxAllowedFixups = 0;273 274 for (MachineBasicBlock &MBB : MF) {275 for (MachineInstr &MI : MBB) {276 if (!isFirstAESPairInstr(MI))277 continue;278 279 // Found an instruction to check the operands of.280 LLVM_DEBUG(dbgs() << "Found AES Pair starting: " << MI);281 assert(MI.getNumExplicitOperands() == 3 && MI.getNumExplicitDefs() == 1 &&282 "Unknown AES Instruction Format. Expected 1 def, 2 uses.");283 284 // A maximum of two fixups should be inserted for each AES pair (one per285 // register use).286 MaxAllowedFixups += 2;287 288 // Inspect all operands, choosing whether to insert a fixup.289 for (MachineOperand &MOp : MI.uses()) {290 SmallPtrSet<MachineInstr *, 1> AllDefs{};291 RDI.getGlobalReachingDefs(&MI, MOp.getReg(), AllDefs);292 293 // Planned Fixup: This should be added to FixupLocsForFn at most once.294 AESFixupLocation NewLoc{&MBB, &MI, &MOp};295 296 // In small functions with loops, this operand may be both a live-in and297 // have definitions within the function itself. These will need a fixup.298 bool IsLiveIn = MF.front().isLiveIn(MOp.getReg());299 300 // If the register doesn't have defining instructions, and is not a301 // live-in, then something is wrong and the fixup must always be302 // inserted to be safe.303 if (!IsLiveIn && AllDefs.size() == 0) {304 LLVM_DEBUG(dbgs()305 << "Fixup Planned: No Defining Instrs found, not live-in: "306 << printReg(MOp.getReg(), TRI) << "\n");307 FixupLocsForFn.emplace_back(NewLoc);308 continue;309 }310 311 auto IsUnsafe = [](MachineInstr *MI) -> bool {312 return !isSafeAESInput(*MI);313 };314 size_t UnsafeCount = llvm::count_if(AllDefs, IsUnsafe);315 316 // If there are no unsafe definitions...317 if (UnsafeCount == 0) {318 // ... and the register is not live-in ...319 if (!IsLiveIn) {320 // ... then skip the fixup.321 LLVM_DEBUG(dbgs() << "No Fixup: Defining instrs are all safe: "322 << printReg(MOp.getReg(), TRI) << "\n");323 continue;324 }325 326 // Otherwise, the only unsafe "definition" is a live-in, so insert the327 // fixup at the start of the function.328 LLVM_DEBUG(dbgs()329 << "Fixup Planned: Live-In (with safe defining instrs): "330 << printReg(MOp.getReg(), TRI) << "\n");331 NewLoc.Block = &MF.front();332 NewLoc.InsertionPt = &*NewLoc.Block->begin();333 LLVM_DEBUG(dbgs() << "Moving Fixup for Live-In to immediately before "334 << *NewLoc.InsertionPt);335 FixupLocsForFn.emplace_back(NewLoc);336 continue;337 }338 339 // If a fixup is needed in more than one place, then the best place to340 // insert it is adjacent to the use rather than introducing a fixup341 // adjacent to each def.342 //343 // FIXME: It might be better to hoist this to the start of the BB, if344 // possible.345 if (IsLiveIn || UnsafeCount > 1) {346 LLVM_DEBUG(dbgs() << "Fixup Planned: Multiple unsafe defining instrs "347 "(including live-ins): "348 << printReg(MOp.getReg(), TRI) << "\n");349 FixupLocsForFn.emplace_back(NewLoc);350 continue;351 }352 353 assert(UnsafeCount == 1 && !IsLiveIn &&354 "At this point, there should be one unsafe defining instrs "355 "and the defined register should not be a live-in.");356 SmallPtrSetIterator<MachineInstr *> It =357 llvm::find_if(AllDefs, IsUnsafe);358 assert(It != AllDefs.end() &&359 "UnsafeCount == 1 but No Unsafe MachineInstr found.");360 MachineInstr *DefMI = *It;361 362 LLVM_DEBUG(363 dbgs() << "Fixup Planned: Found single unsafe defining instrs for "364 << printReg(MOp.getReg(), TRI) << ": " << *DefMI);365 366 // There is one unsafe defining instruction, which needs a fixup. It is367 // generally good to hoist the fixup to be adjacent to the defining368 // instruction rather than the using instruction, as the using369 // instruction may be inside a loop when the defining instruction is370 // not.371 MachineBasicBlock::iterator DefIt = DefMI;372 ++DefIt;373 if (DefIt != DefMI->getParent()->end()) {374 LLVM_DEBUG(dbgs() << "Moving Fixup to immediately after " << *DefMI375 << "And immediately before " << *DefIt);376 NewLoc.Block = DefIt->getParent();377 NewLoc.InsertionPt = &*DefIt;378 }379 380 FixupLocsForFn.emplace_back(NewLoc);381 }382 }383 }384 385 assert(FixupLocsForFn.size() <= MaxAllowedFixups &&386 "Inserted too many fixups for this function.");387 (void)MaxAllowedFixups;388}389 390void ARMFixCortexA57AES1742098::insertAESFixup(391 AESFixupLocation &FixupLoc, const ARMBaseInstrInfo *TII,392 const ARMBaseRegisterInfo *TRI) const {393 MachineOperand *OperandToFixup = FixupLoc.MOp;394 395 assert(OperandToFixup->isReg() && "OperandToFixup must be a register");396 Register RegToFixup = OperandToFixup->getReg();397 398 LLVM_DEBUG(dbgs() << "Inserting VORRq of " << printReg(RegToFixup, TRI)399 << " before: " << *FixupLoc.InsertionPt);400 401 // Insert the new `VORRq qN, qN, qN`. There are a few details here:402 //403 // The uses are marked as killed, even if the original use of OperandToFixup404 // is not killed, as the new instruction is clobbering the register. This is405 // safe even if there are other uses of `qN`, as the VORRq value-wise a no-op406 // (it is inserted for microarchitectural reasons).407 //408 // The def and the uses are still marked as Renamable if the original register409 // was, to avoid having to rummage through all the other uses and defs and410 // unset their renamable bits.411 unsigned Renamable = OperandToFixup->isRenamable() ? RegState::Renamable : 0;412 BuildMI(*FixupLoc.Block, FixupLoc.InsertionPt, DebugLoc(),413 TII->get(ARM::VORRq))414 .addReg(RegToFixup, RegState::Define | Renamable)415 .addReg(RegToFixup, RegState::Kill | Renamable)416 .addReg(RegToFixup, RegState::Kill | Renamable)417 .addImm((uint64_t)ARMCC::AL)418 .addReg(ARM::NoRegister);419}420 421// Factory function used by AArch64TargetMachine to add the pass to422// the passmanager.423FunctionPass *llvm::createARMFixCortexA57AES1742098Pass() {424 return new ARMFixCortexA57AES1742098();425}426