403 lines · cpp
1//===- ARMSLSHardening.cpp - Harden Straight Line Missspeculation ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains a pass to insert code to mitigate against side channel10// vulnerabilities that may happen under straight line miss-speculation.11//12//===----------------------------------------------------------------------===//13 14#include "ARM.h"15#include "ARMInstrInfo.h"16#include "ARMSubtarget.h"17#include "llvm/CodeGen/IndirectThunks.h"18#include "llvm/CodeGen/MachineBasicBlock.h"19#include "llvm/CodeGen/MachineFunction.h"20#include "llvm/CodeGen/MachineFunctionPass.h"21#include "llvm/CodeGen/MachineInstr.h"22#include "llvm/CodeGen/MachineInstrBuilder.h"23#include "llvm/CodeGen/MachineOperand.h"24#include "llvm/IR/DebugLoc.h"25#include <cassert>26 27using namespace llvm;28 29#define DEBUG_TYPE "arm-sls-hardening"30 31#define ARM_SLS_HARDENING_NAME "ARM sls hardening pass"32 33namespace {34 35class ARMSLSHardening : public MachineFunctionPass {36public:37 const TargetInstrInfo *TII;38 const ARMSubtarget *ST;39 40 static char ID;41 42 ARMSLSHardening() : MachineFunctionPass(ID) {43 initializeARMSLSHardeningPass(*PassRegistry::getPassRegistry());44 }45 46 bool runOnMachineFunction(MachineFunction &Fn) override;47 48 StringRef getPassName() const override { return ARM_SLS_HARDENING_NAME; }49 50 void getAnalysisUsage(AnalysisUsage &AU) const override {51 AU.setPreservesCFG();52 MachineFunctionPass::getAnalysisUsage(AU);53 }54 55private:56 bool hardenReturnsAndBRs(MachineBasicBlock &MBB) const;57 bool hardenIndirectCalls(MachineBasicBlock &MBB) const;58 MachineBasicBlock &59 ConvertIndirectCallToIndirectJump(MachineBasicBlock &MBB,60 MachineBasicBlock::iterator) const;61};62 63} // end anonymous namespace64 65char ARMSLSHardening::ID = 0;66 67INITIALIZE_PASS(ARMSLSHardening, "arm-sls-hardening",68 ARM_SLS_HARDENING_NAME, false, false)69 70static void insertSpeculationBarrier(const ARMSubtarget *ST,71 MachineBasicBlock &MBB,72 MachineBasicBlock::iterator MBBI,73 DebugLoc DL,74 bool AlwaysUseISBDSB = false) {75 assert(MBBI != MBB.begin() &&76 "Must not insert SpeculationBarrierEndBB as only instruction in MBB.");77 assert(std::prev(MBBI)->isBarrier() &&78 "SpeculationBarrierEndBB must only follow unconditional control flow "79 "instructions.");80 assert(std::prev(MBBI)->isTerminator() &&81 "SpeculationBarrierEndBB must only follow terminators.");82 const TargetInstrInfo *TII = ST->getInstrInfo();83 assert(ST->hasDataBarrier() || ST->hasSB());84 bool ProduceSB = ST->hasSB() && !AlwaysUseISBDSB;85 unsigned BarrierOpc =86 ProduceSB ? (ST->isThumb() ? ARM::t2SpeculationBarrierSBEndBB87 : ARM::SpeculationBarrierSBEndBB)88 : (ST->isThumb() ? ARM::t2SpeculationBarrierISBDSBEndBB89 : ARM::SpeculationBarrierISBDSBEndBB);90 if (MBBI == MBB.end() || !isSpeculationBarrierEndBBOpcode(MBBI->getOpcode()))91 BuildMI(MBB, MBBI, DL, TII->get(BarrierOpc));92}93 94bool ARMSLSHardening::runOnMachineFunction(MachineFunction &MF) {95 ST = &MF.getSubtarget<ARMSubtarget>();96 TII = MF.getSubtarget().getInstrInfo();97 98 bool Modified = false;99 for (auto &MBB : MF) {100 Modified |= hardenReturnsAndBRs(MBB);101 Modified |= hardenIndirectCalls(MBB);102 }103 104 return Modified;105}106 107bool ARMSLSHardening::hardenReturnsAndBRs(MachineBasicBlock &MBB) const {108 if (!ST->hardenSlsRetBr())109 return false;110 assert(!ST->isThumb1Only());111 bool Modified = false;112 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(), E = MBB.end();113 MachineBasicBlock::iterator NextMBBI;114 for (; MBBI != E; MBBI = NextMBBI) {115 MachineInstr &MI = *MBBI;116 NextMBBI = std::next(MBBI);117 if (isIndirectControlFlowNotComingBack(MI)) {118 assert(MI.isTerminator());119 assert(!TII->isPredicated(MI));120 insertSpeculationBarrier(ST, MBB, std::next(MBBI), MI.getDebugLoc());121 Modified = true;122 }123 }124 return Modified;125}126 127static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";128 129static const struct ThunkNameRegMode {130 const char* Name;131 Register Reg;132 bool isThumb;133} SLSBLRThunks[] = {134 {"__llvm_slsblr_thunk_arm_r0", ARM::R0, false},135 {"__llvm_slsblr_thunk_arm_r1", ARM::R1, false},136 {"__llvm_slsblr_thunk_arm_r2", ARM::R2, false},137 {"__llvm_slsblr_thunk_arm_r3", ARM::R3, false},138 {"__llvm_slsblr_thunk_arm_r4", ARM::R4, false},139 {"__llvm_slsblr_thunk_arm_r5", ARM::R5, false},140 {"__llvm_slsblr_thunk_arm_r6", ARM::R6, false},141 {"__llvm_slsblr_thunk_arm_r7", ARM::R7, false},142 {"__llvm_slsblr_thunk_arm_r8", ARM::R8, false},143 {"__llvm_slsblr_thunk_arm_r9", ARM::R9, false},144 {"__llvm_slsblr_thunk_arm_r10", ARM::R10, false},145 {"__llvm_slsblr_thunk_arm_r11", ARM::R11, false},146 {"__llvm_slsblr_thunk_arm_sp", ARM::SP, false},147 {"__llvm_slsblr_thunk_arm_pc", ARM::PC, false},148 {"__llvm_slsblr_thunk_thumb_r0", ARM::R0, true},149 {"__llvm_slsblr_thunk_thumb_r1", ARM::R1, true},150 {"__llvm_slsblr_thunk_thumb_r2", ARM::R2, true},151 {"__llvm_slsblr_thunk_thumb_r3", ARM::R3, true},152 {"__llvm_slsblr_thunk_thumb_r4", ARM::R4, true},153 {"__llvm_slsblr_thunk_thumb_r5", ARM::R5, true},154 {"__llvm_slsblr_thunk_thumb_r6", ARM::R6, true},155 {"__llvm_slsblr_thunk_thumb_r7", ARM::R7, true},156 {"__llvm_slsblr_thunk_thumb_r8", ARM::R8, true},157 {"__llvm_slsblr_thunk_thumb_r9", ARM::R9, true},158 {"__llvm_slsblr_thunk_thumb_r10", ARM::R10, true},159 {"__llvm_slsblr_thunk_thumb_r11", ARM::R11, true},160 {"__llvm_slsblr_thunk_thumb_sp", ARM::SP, true},161 {"__llvm_slsblr_thunk_thumb_pc", ARM::PC, true},162};163 164// An enum for tracking whether Arm and Thumb thunks have been inserted into the165// current module so far.166enum ArmInsertedThunks { NoThunk = 0, ArmThunk = 1, ThumbThunk = 2 };167 168inline ArmInsertedThunks &operator|=(ArmInsertedThunks &X,169 ArmInsertedThunks Y) {170 return X = static_cast<ArmInsertedThunks>(X | Y);171}172 173namespace {174struct SLSBLRThunkInserter175 : ThunkInserter<SLSBLRThunkInserter, ArmInsertedThunks> {176 const char *getThunkPrefix() { return SLSBLRNamePrefix; }177 bool mayUseThunk(const MachineFunction &MF) {178 ComdatThunks &= !MF.getSubtarget<ARMSubtarget>().hardenSlsNoComdat();179 return MF.getSubtarget<ARMSubtarget>().hardenSlsBlr();180 }181 ArmInsertedThunks insertThunks(MachineModuleInfo &MMI, MachineFunction &MF,182 ArmInsertedThunks InsertedThunks);183 void populateThunk(MachineFunction &MF);184 185private:186 bool ComdatThunks = true;187};188} // namespace189 190ArmInsertedThunks191SLSBLRThunkInserter::insertThunks(MachineModuleInfo &MMI, MachineFunction &MF,192 ArmInsertedThunks InsertedThunks) {193 if ((InsertedThunks & ArmThunk &&194 !MF.getSubtarget<ARMSubtarget>().isThumb()) ||195 (InsertedThunks & ThumbThunk &&196 MF.getSubtarget<ARMSubtarget>().isThumb()))197 return NoThunk;198 // FIXME: It probably would be possible to filter which thunks to produce199 // based on which registers are actually used in indirect calls in this200 // function. But would that be a worthwhile optimization?201 const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();202 for (auto T : SLSBLRThunks)203 if (ST->isThumb() == T.isThumb)204 createThunkFunction(MMI, T.Name, ComdatThunks,205 T.isThumb ? "+thumb-mode" : "");206 return ST->isThumb() ? ThumbThunk : ArmThunk;207}208 209void SLSBLRThunkInserter::populateThunk(MachineFunction &MF) {210 assert(MF.getFunction().hasComdat() == ComdatThunks &&211 "ComdatThunks value changed since MF creation");212 // FIXME: How to better communicate Register number, rather than through213 // name and lookup table?214 assert(MF.getName().starts_with(getThunkPrefix()));215 auto ThunkIt = llvm::find_if(216 SLSBLRThunks, [&MF](auto T) { return T.Name == MF.getName(); });217 assert(ThunkIt != std::end(SLSBLRThunks));218 Register ThunkReg = ThunkIt->Reg;219 bool isThumb = ThunkIt->isThumb;220 221 const TargetInstrInfo *TII = MF.getSubtarget<ARMSubtarget>().getInstrInfo();222 MachineBasicBlock *Entry = &MF.front();223 Entry->clear();224 225 // These thunks need to consist of the following instructions:226 // __llvm_slsblr_thunk_(arm/thumb)_rN:227 // bx rN228 // barrierInsts229 Entry->addLiveIn(ThunkReg);230 if (isThumb)231 BuildMI(Entry, DebugLoc(), TII->get(ARM::tBX))232 .addReg(ThunkReg)233 .add(predOps(ARMCC::AL));234 else235 BuildMI(Entry, DebugLoc(), TII->get(ARM::BX))236 .addReg(ThunkReg);237 238 // Make sure the thunks do not make use of the SB extension in case there is239 // a function somewhere that will call to it that for some reason disabled240 // the SB extension locally on that function, even though it's enabled for241 // the module otherwise. Therefore set AlwaysUseISBSDB to true.242 insertSpeculationBarrier(&MF.getSubtarget<ARMSubtarget>(), *Entry,243 Entry->end(), DebugLoc(), true /*AlwaysUseISBDSB*/);244}245 246MachineBasicBlock &ARMSLSHardening::ConvertIndirectCallToIndirectJump(247 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {248 // Transform an indirect call to an indirect jump as follows:249 // Before:250 // |-----------------------------|251 // | ... |252 // | instI |253 // | BLX rN |254 // | instJ |255 // | ... |256 // |-----------------------------|257 //258 // After:259 // |---------- -------------------------|260 // | ... |261 // | instI |262 // | *call* __llvm_slsblr_thunk_mode_xN |263 // | instJ |264 // | ... |265 // |--------------------------------------|266 //267 // __llvm_slsblr_thunk_mode_xN:268 // |-----------------------------|269 // | BX rN |270 // | barrierInsts |271 // |-----------------------------|272 //273 // The __llvm_slsblr_thunk_mode_xN thunks are created by the274 // SLSBLRThunkInserter.275 // This function merely needs to transform an indirect call to a direct call276 // to __llvm_slsblr_thunk_xN.277 MachineInstr &IndirectCall = *MBBI;278 assert(isIndirectCall(IndirectCall) && !IndirectCall.isReturn());279 int RegOpIdxOnIndirectCall = -1;280 bool isThumb;281 switch (IndirectCall.getOpcode()) {282 case ARM::BLX: // !isThumb2283 case ARM::BLX_noip: // !isThumb2284 isThumb = false;285 RegOpIdxOnIndirectCall = 0;286 break;287 case ARM::tBLXr: // isThumb2288 case ARM::tBLXr_noip: // isThumb2289 isThumb = true;290 RegOpIdxOnIndirectCall = 2;291 break;292 default:293 llvm_unreachable("unhandled Indirect Call");294 }295 296 Register Reg = IndirectCall.getOperand(RegOpIdxOnIndirectCall).getReg();297 // Since linkers are allowed to clobber R12 on function calls, the above298 // mitigation only works if the original indirect call instruction was not299 // using R12. Code generation before must make sure that no indirect call300 // using R12 was produced if the mitigation is enabled.301 // Also, the transformation is incorrect if the indirect call uses LR, so302 // also have to avoid that.303 assert(Reg != ARM::R12 && Reg != ARM::LR);304 bool RegIsKilled = IndirectCall.getOperand(RegOpIdxOnIndirectCall).isKill();305 306 DebugLoc DL = IndirectCall.getDebugLoc();307 308 MachineFunction &MF = *MBBI->getMF();309 auto ThunkIt = llvm::find_if(SLSBLRThunks, [Reg, isThumb](auto T) {310 return T.Reg == Reg && T.isThumb == isThumb;311 });312 assert(ThunkIt != std::end(SLSBLRThunks));313 Module *M = MF.getFunction().getParent();314 const GlobalValue *GV = cast<GlobalValue>(M->getNamedValue(ThunkIt->Name));315 316 MachineInstr *BL =317 isThumb ? BuildMI(MBB, MBBI, DL, TII->get(ARM::tBL))318 .addImm(IndirectCall.getOperand(0).getImm())319 .addReg(IndirectCall.getOperand(1).getReg())320 .addGlobalAddress(GV)321 : BuildMI(MBB, MBBI, DL, TII->get(ARM::BL)).addGlobalAddress(GV);322 323 // Now copy the implicit operands from IndirectCall to BL and copy other324 // necessary info.325 // However, both IndirectCall and BL instructions implictly use SP and326 // implicitly define LR. Blindly copying implicit operands would result in SP327 // and LR operands to be present multiple times. While this may not be too328 // much of an issue, let's avoid that for cleanliness, by removing those329 // implicit operands from the BL created above before we copy over all330 // implicit operands from the IndirectCall.331 int ImpLROpIdx = -1;332 int ImpSPOpIdx = -1;333 for (unsigned OpIdx = BL->getNumExplicitOperands();334 OpIdx < BL->getNumOperands(); OpIdx++) {335 MachineOperand Op = BL->getOperand(OpIdx);336 if (!Op.isReg())337 continue;338 if (Op.getReg() == ARM::LR && Op.isDef())339 ImpLROpIdx = OpIdx;340 if (Op.getReg() == ARM::SP && !Op.isDef())341 ImpSPOpIdx = OpIdx;342 }343 assert(ImpLROpIdx != -1);344 assert(ImpSPOpIdx != -1);345 int FirstOpIdxToRemove = std::max(ImpLROpIdx, ImpSPOpIdx);346 int SecondOpIdxToRemove = std::min(ImpLROpIdx, ImpSPOpIdx);347 BL->removeOperand(FirstOpIdxToRemove);348 BL->removeOperand(SecondOpIdxToRemove);349 // Now copy over the implicit operands from the original IndirectCall350 BL->copyImplicitOps(MF, IndirectCall);351 MF.moveAdditionalCallInfo(&IndirectCall, BL);352 // Also add the register called in the IndirectCall as being used in the353 // called thunk.354 BL->addOperand(MachineOperand::CreateReg(Reg, false /*isDef*/, true /*isImp*/,355 RegIsKilled /*isKill*/));356 // Remove IndirectCallinstruction357 MBB.erase(MBBI);358 return MBB;359}360 361bool ARMSLSHardening::hardenIndirectCalls(MachineBasicBlock &MBB) const {362 if (!ST->hardenSlsBlr())363 return false;364 bool Modified = false;365 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();366 MachineBasicBlock::iterator NextMBBI;367 for (; MBBI != E; MBBI = NextMBBI) {368 MachineInstr &MI = *MBBI;369 NextMBBI = std::next(MBBI);370 // Tail calls are both indirect calls and "returns".371 // They are also indirect jumps, so should be handled by sls-harden-retbr,372 // rather than sls-harden-blr.373 if (isIndirectCall(MI) && !MI.isReturn()) {374 ConvertIndirectCallToIndirectJump(MBB, MBBI);375 Modified = true;376 }377 }378 return Modified;379}380 381 382 383FunctionPass *llvm::createARMSLSHardeningPass() {384 return new ARMSLSHardening();385}386 387namespace {388class ARMIndirectThunks : public ThunkInserterPass<SLSBLRThunkInserter> {389public:390 static char ID;391 392 ARMIndirectThunks() : ThunkInserterPass(ID) {}393 394 StringRef getPassName() const override { return "ARM Indirect Thunks"; }395};396} // end anonymous namespace397 398char ARMIndirectThunks::ID = 0;399 400FunctionPass *llvm::createARMIndirectThunks() {401 return new ARMIndirectThunks();402}403