992 lines · cpp
1//===--------- aarch32.cpp - Generic JITLink arm/thumb utilities ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Generic utilities for graphs representing arm/thumb objects.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/ExecutionEngine/JITLink/aarch32.h"14 15#include "llvm/ADT/StringExtras.h"16#include "llvm/ExecutionEngine/JITLink/JITLink.h"17#include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"18#include "llvm/Support/Compiler.h"19#include "llvm/Support/Endian.h"20#include "llvm/Support/ManagedStatic.h"21#include "llvm/Support/MathExtras.h"22 23#define DEBUG_TYPE "jitlink"24 25namespace llvm {26namespace jitlink {27namespace aarch32 {28 29/// Check whether the given target flags are set for this Symbol.30bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags) {31 return Sym.getTargetFlags() & Flags;32}33 34/// Encode 22-bit immediate value for branch instructions without J1J2 range35/// extension (formats B T4, BL T1 and BLX T2).36///37/// 00000:Imm11H:Imm11L:0 -> [ 00000:Imm11H, 00000:Imm11L ]38/// J1^ ^J2 will always be 139///40HalfWords encodeImmBT4BlT1BlxT2(int64_t Value) {41 constexpr uint32_t J1J2 = 0x2800;42 uint32_t Imm11H = (Value >> 12) & 0x07ff;43 uint32_t Imm11L = (Value >> 1) & 0x07ff;44 return HalfWords{Imm11H, Imm11L | J1J2};45}46 47/// Decode 22-bit immediate value for branch instructions without J1J2 range48/// extension (formats B T4, BL T1 and BLX T2).49///50/// [ 00000:Imm11H, 00000:Imm11L ] -> 00000:Imm11H:Imm11L:051/// J1^ ^J2 will always be 152///53int64_t decodeImmBT4BlT1BlxT2(uint32_t Hi, uint32_t Lo) {54 uint32_t Imm11H = Hi & 0x07ff;55 uint32_t Imm11L = Lo & 0x07ff;56 return SignExtend64<22>(Imm11H << 12 | Imm11L << 1);57}58 59/// Encode 25-bit immediate value for branch instructions with J1J2 range60/// extension (formats B T4, BL T1 and BLX T2).61///62/// S:I1:I2:Imm10:Imm11:0 -> [ 00000:S:Imm10, 00:J1:0:J2:Imm11 ]63///64LLVM_ABI HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {65 uint32_t S = (Value >> 14) & 0x0400;66 uint32_t J1 = (((~(Value >> 10)) ^ (Value >> 11)) & 0x2000);67 uint32_t J2 = (((~(Value >> 11)) ^ (Value >> 13)) & 0x0800);68 uint32_t Imm10 = (Value >> 12) & 0x03ff;69 uint32_t Imm11 = (Value >> 1) & 0x07ff;70 return HalfWords{S | Imm10, J1 | J2 | Imm11};71}72 73/// Decode 25-bit immediate value for branch instructions with J1J2 range74/// extension (formats B T4, BL T1 and BLX T2).75///76/// [ 00000:S:Imm10, 00:J1:0:J2:Imm11] -> S:I1:I2:Imm10:Imm11:077///78LLVM_ABI int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {79 uint32_t S = Hi & 0x0400;80 uint32_t I1 = ~((Lo ^ (Hi << 3)) << 10) & 0x00800000;81 uint32_t I2 = ~((Lo ^ (Hi << 1)) << 11) & 0x00400000;82 uint32_t Imm10 = Hi & 0x03ff;83 uint32_t Imm11 = Lo & 0x07ff;84 return SignExtend64<25>(S << 14 | I1 | I2 | Imm10 << 12 | Imm11 << 1);85}86 87/// Encode 26-bit immediate value for branch instructions88/// (formats B A1, BL A1 and BLX A2).89///90/// Imm24:00 -> 00000000:Imm2491///92LLVM_ABI uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {93 return (Value >> 2) & 0x00ffffff;94}95 96/// Decode 26-bit immediate value for branch instructions97/// (formats B A1, BL A1 and BLX A2).98///99/// 00000000:Imm24 -> Imm24:00100///101LLVM_ABI int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {102 return SignExtend64<26>((Value & 0x00ffffff) << 2);103}104 105/// Encode 16-bit immediate value for move instruction formats MOVT T1 and106/// MOVW T3.107///108/// Imm4:Imm1:Imm3:Imm8 -> [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ]109///110LLVM_ABI HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {111 uint32_t Imm4 = (Value >> 12) & 0x0f;112 uint32_t Imm1 = (Value >> 11) & 0x01;113 uint32_t Imm3 = (Value >> 8) & 0x07;114 uint32_t Imm8 = Value & 0xff;115 return HalfWords{Imm1 << 10 | Imm4, Imm3 << 12 | Imm8};116}117 118/// Decode 16-bit immediate value from move instruction formats MOVT T1 and119/// MOVW T3.120///121/// [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] -> Imm4:Imm1:Imm3:Imm8122///123LLVM_ABI uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {124 uint32_t Imm4 = Hi & 0x0f;125 uint32_t Imm1 = (Hi >> 10) & 0x01;126 uint32_t Imm3 = (Lo >> 12) & 0x07;127 uint32_t Imm8 = Lo & 0xff;128 uint32_t Imm16 = Imm4 << 12 | Imm1 << 11 | Imm3 << 8 | Imm8;129 assert(Imm16 <= 0xffff && "Decoded value out-of-range");130 return Imm16;131}132 133/// Encode register ID for instruction formats MOVT T1 and MOVW T3.134///135/// Rd4 -> [0000000000000000, 0000:Rd4:00000000]136///137LLVM_ABI HalfWords encodeRegMovtT1MovwT3(int64_t Value) {138 uint32_t Rd4 = (Value & 0x0f) << 8;139 return HalfWords{0, Rd4};140}141 142/// Decode register ID from instruction formats MOVT T1 and MOVW T3.143///144/// [0000000000000000, 0000:Rd4:00000000] -> Rd4145///146LLVM_ABI int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {147 uint32_t Rd4 = (Lo >> 8) & 0x0f;148 return Rd4;149}150 151/// Encode 16-bit immediate value for move instruction formats MOVT A1 and152/// MOVW A2.153///154/// Imm4:Imm12 -> 000000000000:Imm4:0000:Imm12155///156LLVM_ABI uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {157 uint32_t Imm4 = (Value >> 12) & 0x0f;158 uint32_t Imm12 = Value & 0x0fff;159 return (Imm4 << 16) | Imm12;160}161 162/// Decode 16-bit immediate value for move instruction formats MOVT A1 and163/// MOVW A2.164///165/// 000000000000:Imm4:0000:Imm12 -> Imm4:Imm12166///167LLVM_ABI uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {168 uint32_t Imm4 = (Value >> 16) & 0x0f;169 uint32_t Imm12 = Value & 0x0fff;170 return (Imm4 << 12) | Imm12;171}172 173/// Encode register ID for instruction formats MOVT A1 and174/// MOVW A2.175///176/// Rd4 -> 0000000000000000:Rd4:000000000000177///178LLVM_ABI uint32_t encodeRegMovtA1MovwA2(int64_t Value) {179 uint32_t Rd4 = (Value & 0x00000f) << 12;180 return Rd4;181}182 183/// Decode register ID for instruction formats MOVT A1 and184/// MOVW A2.185///186/// 0000000000000000:Rd4:000000000000 -> Rd4187///188LLVM_ABI int64_t decodeRegMovtA1MovwA2(uint64_t Value) {189 uint32_t Rd4 = (Value >> 12) & 0x00000f;190 return Rd4;191}192 193namespace {194 195/// 32-bit Thumb instructions are stored as two little-endian halfwords.196/// An instruction at address A encodes bytes A+1, A in the first halfword (Hi),197/// followed by bytes A+3, A+2 in the second halfword (Lo).198struct WritableThumbRelocation {199 /// Create a writable reference to a Thumb32 fixup.200 WritableThumbRelocation(char *FixupPtr)201 : Hi{*reinterpret_cast<support::ulittle16_t *>(FixupPtr)},202 Lo{*reinterpret_cast<support::ulittle16_t *>(FixupPtr + 2)} {}203 204 support::ulittle16_t &Hi; // First halfword205 support::ulittle16_t &Lo; // Second halfword206};207 208struct ThumbRelocation {209 /// Create a read-only reference to a Thumb32 fixup.210 ThumbRelocation(const char *FixupPtr)211 : Hi{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr)},212 Lo{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr + 2)} {}213 214 /// Create a read-only Thumb32 fixup from a writeable one.215 ThumbRelocation(WritableThumbRelocation &Writable)216 : Hi{Writable.Hi}, Lo(Writable.Lo) {}217 218 const support::ulittle16_t &Hi; // First halfword219 const support::ulittle16_t &Lo; // Second halfword220};221 222struct WritableArmRelocation {223 WritableArmRelocation(char *FixupPtr)224 : Wd{*reinterpret_cast<support::ulittle32_t *>(FixupPtr)} {}225 226 support::ulittle32_t &Wd;227};228 229struct ArmRelocation {230 ArmRelocation(const char *FixupPtr)231 : Wd{*reinterpret_cast<const support::ulittle32_t *>(FixupPtr)} {}232 233 ArmRelocation(WritableArmRelocation &Writable) : Wd{Writable.Wd} {}234 235 const support::ulittle32_t &Wd;236};237 238Error makeUnexpectedOpcodeError(const LinkGraph &G, const ThumbRelocation &R,239 Edge::Kind Kind) {240 return make_error<JITLinkError>(241 formatv("Invalid opcode [ {0:x4}, {1:x4} ] for relocation: {2}",242 R.Hi.value(), R.Lo.value(), G.getEdgeKindName(Kind)));243}244 245Error makeUnexpectedOpcodeError(const LinkGraph &G, const ArmRelocation &R,246 Edge::Kind Kind) {247 return make_error<JITLinkError>(248 formatv("Invalid opcode {0:x8} for relocation: {1}", R.Wd.value(),249 G.getEdgeKindName(Kind)));250}251 252template <EdgeKind_aarch32 K> constexpr bool isArm() {253 return FirstArmRelocation <= K && K <= LastArmRelocation;254}255template <EdgeKind_aarch32 K> constexpr bool isThumb() {256 return FirstThumbRelocation <= K && K <= LastThumbRelocation;257}258 259template <EdgeKind_aarch32 K> static bool checkOpcodeArm(uint32_t Wd) {260 return (Wd & FixupInfo<K>::OpcodeMask) == FixupInfo<K>::Opcode;261}262 263template <EdgeKind_aarch32 K>264static bool checkOpcodeThumb(uint16_t Hi, uint16_t Lo) {265 return (Hi & FixupInfo<K>::OpcodeMask.Hi) == FixupInfo<K>::Opcode.Hi &&266 (Lo & FixupInfo<K>::OpcodeMask.Lo) == FixupInfo<K>::Opcode.Lo;267}268 269class FixupInfoTable {270 static constexpr size_t Items = LastRelocation + 1;271 272public:273 FixupInfoTable() {274 populateEntries<FirstArmRelocation, LastArmRelocation>();275 populateEntries<FirstThumbRelocation, LastThumbRelocation>();276 }277 278 const FixupInfoBase *getEntry(Edge::Kind K) {279 assert(K < Data.size() && "Index out of bounds");280 return Data.at(K).get();281 }282 283private:284 template <EdgeKind_aarch32 K, EdgeKind_aarch32 LastK> void populateEntries() {285 assert(K < Data.size() && "Index out of range");286 assert(Data.at(K) == nullptr && "Initialized entries are immutable");287 Data[K] = initEntry<K>();288 if constexpr (K < LastK) {289 constexpr auto Next = static_cast<EdgeKind_aarch32>(K + 1);290 populateEntries<Next, LastK>();291 }292 }293 294 template <EdgeKind_aarch32 K>295 static std::unique_ptr<FixupInfoBase> initEntry() {296 auto Entry = std::make_unique<FixupInfo<K>>();297 static_assert(isArm<K>() != isThumb<K>(), "Classes are mutually exclusive");298 if constexpr (isArm<K>())299 Entry->checkOpcode = checkOpcodeArm<K>;300 if constexpr (isThumb<K>())301 Entry->checkOpcode = checkOpcodeThumb<K>;302 return Entry;303 }304 305private:306 std::array<std::unique_ptr<FixupInfoBase>, Items> Data;307};308 309ManagedStatic<FixupInfoTable> DynFixupInfos;310 311} // namespace312 313static Error checkOpcode(LinkGraph &G, const ArmRelocation &R,314 Edge::Kind Kind) {315 assert(Kind >= FirstArmRelocation && Kind <= LastArmRelocation &&316 "Edge kind must be Arm relocation");317 const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind);318 const FixupInfoArm &Info = *static_cast<const FixupInfoArm *>(Entry);319 assert(Info.checkOpcode && "Opcode check is mandatory for Arm edges");320 if (!Info.checkOpcode(R.Wd))321 return makeUnexpectedOpcodeError(G, R, Kind);322 323 return Error::success();324}325 326static Error checkOpcode(LinkGraph &G, const ThumbRelocation &R,327 Edge::Kind Kind) {328 assert(Kind >= FirstThumbRelocation && Kind <= LastThumbRelocation &&329 "Edge kind must be Thumb relocation");330 const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind);331 const FixupInfoThumb &Info = *static_cast<const FixupInfoThumb *>(Entry);332 assert(Info.checkOpcode && "Opcode check is mandatory for Thumb edges");333 if (!Info.checkOpcode(R.Hi, R.Lo))334 return makeUnexpectedOpcodeError(G, R, Kind);335 336 return Error::success();337}338 339const FixupInfoBase *FixupInfoBase::getDynFixupInfo(Edge::Kind K) {340 return DynFixupInfos->getEntry(K);341}342 343template <EdgeKind_aarch32 Kind>344bool checkRegister(const ThumbRelocation &R, HalfWords Reg) {345 uint16_t Hi = R.Hi & FixupInfo<Kind>::RegMask.Hi;346 uint16_t Lo = R.Lo & FixupInfo<Kind>::RegMask.Lo;347 return Hi == Reg.Hi && Lo == Reg.Lo;348}349 350template <EdgeKind_aarch32 Kind>351bool checkRegister(const ArmRelocation &R, uint32_t Reg) {352 uint32_t Wd = R.Wd & FixupInfo<Kind>::RegMask;353 return Wd == Reg;354}355 356template <EdgeKind_aarch32 Kind>357void writeRegister(WritableThumbRelocation &R, HalfWords Reg) {358 static constexpr HalfWords Mask = FixupInfo<Kind>::RegMask;359 assert((Mask.Hi & Reg.Hi) == Reg.Hi && (Mask.Lo & Reg.Lo) == Reg.Lo &&360 "Value bits exceed bit range of given mask");361 R.Hi = (R.Hi & ~Mask.Hi) | Reg.Hi;362 R.Lo = (R.Lo & ~Mask.Lo) | Reg.Lo;363}364 365template <EdgeKind_aarch32 Kind>366void writeRegister(WritableArmRelocation &R, uint32_t Reg) {367 static constexpr uint32_t Mask = FixupInfo<Kind>::RegMask;368 assert((Mask & Reg) == Reg && "Value bits exceed bit range of given mask");369 R.Wd = (R.Wd & ~Mask) | Reg;370}371 372template <EdgeKind_aarch32 Kind>373void writeImmediate(WritableThumbRelocation &R, HalfWords Imm) {374 static constexpr HalfWords Mask = FixupInfo<Kind>::ImmMask;375 assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Lo & Imm.Lo) == Imm.Lo &&376 "Value bits exceed bit range of given mask");377 R.Hi = (R.Hi & ~Mask.Hi) | Imm.Hi;378 R.Lo = (R.Lo & ~Mask.Lo) | Imm.Lo;379}380 381template <EdgeKind_aarch32 Kind>382void writeImmediate(WritableArmRelocation &R, uint32_t Imm) {383 static constexpr uint32_t Mask = FixupInfo<Kind>::ImmMask;384 assert((Mask & Imm) == Imm && "Value bits exceed bit range of given mask");385 R.Wd = (R.Wd & ~Mask) | Imm;386}387 388Expected<int64_t> readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset,389 Edge::Kind Kind) {390 endianness Endian = G.getEndianness();391 const char *BlockWorkingMem = B.getContent().data();392 const char *FixupPtr = BlockWorkingMem + Offset;393 394 switch (Kind) {395 case Data_Delta32:396 case Data_Pointer32:397 case Data_RequestGOTAndTransformToDelta32:398 return SignExtend64<32>(support::endian::read32(FixupPtr, Endian));399 case Data_PRel31:400 return SignExtend64<31>(support::endian::read32(FixupPtr, Endian));401 default:402 return make_error<JITLinkError>(403 "In graph " + G.getName() + ", section " + B.getSection().getName() +404 " can not read implicit addend for aarch32 edge kind " +405 G.getEdgeKindName(Kind));406 }407}408 409Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, Edge::OffsetT Offset,410 Edge::Kind Kind) {411 ArmRelocation R(B.getContent().data() + Offset);412 if (Error Err = checkOpcode(G, R, Kind))413 return std::move(Err);414 415 switch (Kind) {416 case Arm_Call:417 case Arm_Jump24:418 return decodeImmBA1BlA1BlxA2(R.Wd);419 420 case Arm_MovtAbs:421 case Arm_MovwAbsNC:422 return decodeImmMovtA1MovwA2(R.Wd);423 424 default:425 return make_error<JITLinkError>(426 "In graph " + G.getName() + ", section " + B.getSection().getName() +427 " can not read implicit addend for aarch32 edge kind " +428 G.getEdgeKindName(Kind));429 }430}431 432Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, Edge::OffsetT Offset,433 Edge::Kind Kind, const ArmConfig &ArmCfg) {434 ThumbRelocation R(B.getContent().data() + Offset);435 if (Error Err = checkOpcode(G, R, Kind))436 return std::move(Err);437 438 switch (Kind) {439 case Thumb_Call:440 case Thumb_Jump24:441 return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)442 ? decodeImmBT4BlT1BlxT2_J1J2(R.Hi, R.Lo)443 : decodeImmBT4BlT1BlxT2(R.Hi, R.Lo);444 445 case Thumb_MovwAbsNC:446 case Thumb_MovwPrelNC:447 // Initial addend is interpreted as a signed value448 return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo));449 450 case Thumb_MovtAbs:451 case Thumb_MovtPrel:452 // Initial addend is interpreted as a signed value453 return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo));454 455 default:456 return make_error<JITLinkError>(457 "In graph " + G.getName() + ", section " + B.getSection().getName() +458 " can not read implicit addend for aarch32 edge kind " +459 G.getEdgeKindName(Kind));460 }461}462 463Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) {464 using namespace support;465 466 char *BlockWorkingMem = B.getAlreadyMutableContent().data();467 char *FixupPtr = BlockWorkingMem + E.getOffset();468 469 Edge::Kind Kind = E.getKind();470 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();471 int64_t Addend = E.getAddend();472 Symbol &TargetSymbol = E.getTarget();473 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();474 475 // Data relocations have alignment 1, size 4 (except R_ARM_ABS8 and476 // R_ARM_ABS16) and write the full 32-bit result (except R_ARM_PREL31).477 switch (Kind) {478 case Data_Delta32: {479 int64_t Value = TargetAddress - FixupAddress + Addend;480 if (!isInt<32>(Value))481 return makeTargetOutOfRangeError(G, B, E);482 if (LLVM_LIKELY(G.getEndianness() == endianness::little))483 endian::write32le(FixupPtr, Value);484 else485 endian::write32be(FixupPtr, Value);486 return Error::success();487 }488 case Data_Pointer32: {489 int64_t Value = TargetAddress + Addend;490 if (!isUInt<32>(Value))491 return makeTargetOutOfRangeError(G, B, E);492 if (LLVM_LIKELY(G.getEndianness() == endianness::little))493 endian::write32le(FixupPtr, Value);494 else495 endian::write32be(FixupPtr, Value);496 return Error::success();497 }498 case Data_PRel31: {499 int64_t Value = TargetAddress - FixupAddress + Addend;500 if (!isInt<31>(Value))501 return makeTargetOutOfRangeError(G, B, E);502 if (LLVM_LIKELY(G.getEndianness() == endianness::little)) {503 uint32_t MSB = endian::read32le(FixupPtr) & 0x80000000;504 endian::write32le(FixupPtr, MSB | (Value & ~0x80000000));505 } else {506 uint32_t MSB = endian::read32be(FixupPtr) & 0x80000000;507 endian::write32be(FixupPtr, MSB | (Value & ~0x80000000));508 }509 return Error::success();510 }511 case Data_RequestGOTAndTransformToDelta32:512 llvm_unreachable("Should be transformed");513 default:514 return make_error<JITLinkError>(515 "In graph " + G.getName() + ", section " + B.getSection().getName() +516 " encountered unfixable aarch32 edge kind " +517 G.getEdgeKindName(E.getKind()));518 }519}520 521Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E) {522 WritableArmRelocation R(B.getAlreadyMutableContent().data() + E.getOffset());523 Edge::Kind Kind = E.getKind();524 if (Error Err = checkOpcode(G, R, Kind))525 return Err;526 527 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();528 int64_t Addend = E.getAddend();529 Symbol &TargetSymbol = E.getTarget();530 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();531 532 switch (Kind) {533 case Arm_Jump24: {534 if (hasTargetFlags(TargetSymbol, ThumbSymbol))535 return make_error<JITLinkError>("Branch relocation needs interworking "536 "stub when bridging to Thumb: " +537 StringRef(G.getEdgeKindName(Kind)));538 539 int64_t Value = TargetAddress - FixupAddress + Addend;540 541 if (!isInt<26>(Value))542 return makeTargetOutOfRangeError(G, B, E);543 writeImmediate<Arm_Jump24>(R, encodeImmBA1BlA1BlxA2(Value));544 545 return Error::success();546 }547 case Arm_Call: {548 if ((R.Wd & FixupInfo<Arm_Call>::CondMask) !=549 FixupInfo<Arm_Call>::Unconditional)550 return make_error<JITLinkError>("Relocation expects an unconditional "551 "BL/BLX branch instruction: " +552 StringRef(G.getEdgeKindName(Kind)));553 554 int64_t Value = TargetAddress - FixupAddress + Addend;555 556 // The call instruction itself is Arm. The call destination can either be557 // Thumb or Arm. We use BL to stay in Arm and BLX to change to Thumb.558 bool TargetIsThumb = hasTargetFlags(TargetSymbol, ThumbSymbol);559 bool InstrIsBlx = (~R.Wd & FixupInfo<Arm_Call>::BitBlx) == 0;560 if (TargetIsThumb != InstrIsBlx) {561 if (LLVM_LIKELY(TargetIsThumb)) {562 // Change opcode BL -> BLX563 R.Wd = R.Wd | FixupInfo<Arm_Call>::BitBlx;564 R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitH;565 } else {566 // Change opcode BLX -> BL567 R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitBlx;568 }569 }570 571 if (!isInt<26>(Value))572 return makeTargetOutOfRangeError(G, B, E);573 writeImmediate<Arm_Call>(R, encodeImmBA1BlA1BlxA2(Value));574 575 return Error::success();576 }577 case Arm_MovwAbsNC: {578 uint16_t Value = (TargetAddress + Addend) & 0xffff;579 writeImmediate<Arm_MovwAbsNC>(R, encodeImmMovtA1MovwA2(Value));580 return Error::success();581 }582 case Arm_MovtAbs: {583 uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;584 writeImmediate<Arm_MovtAbs>(R, encodeImmMovtA1MovwA2(Value));585 return Error::success();586 }587 default:588 return make_error<JITLinkError>(589 "In graph " + G.getName() + ", section " + B.getSection().getName() +590 " encountered unfixable aarch32 edge kind " +591 G.getEdgeKindName(E.getKind()));592 }593}594 595Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,596 const ArmConfig &ArmCfg) {597 WritableThumbRelocation R(B.getAlreadyMutableContent().data() +598 E.getOffset());599 Edge::Kind Kind = E.getKind();600 if (Error Err = checkOpcode(G, R, Kind))601 return Err;602 603 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();604 int64_t Addend = E.getAddend();605 Symbol &TargetSymbol = E.getTarget();606 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();607 608 switch (Kind) {609 case Thumb_Jump24: {610 if (!hasTargetFlags(TargetSymbol, ThumbSymbol))611 return make_error<JITLinkError>("Branch relocation needs interworking "612 "stub when bridging to ARM: " +613 StringRef(G.getEdgeKindName(Kind)));614 615 int64_t Value = TargetAddress - FixupAddress + Addend;616 if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {617 if (!isInt<25>(Value))618 return makeTargetOutOfRangeError(G, B, E);619 writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2_J1J2(Value));620 } else {621 if (!isInt<22>(Value))622 return makeTargetOutOfRangeError(G, B, E);623 writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2(Value));624 }625 626 return Error::success();627 }628 629 case Thumb_Call: {630 int64_t Value = TargetAddress - FixupAddress + Addend;631 632 // The call instruction itself is Thumb. The call destination can either be633 // Thumb or Arm. We use BL to stay in Thumb and BLX to change to Arm.634 bool TargetIsArm = !hasTargetFlags(TargetSymbol, ThumbSymbol);635 bool InstrIsBlx = (R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) == 0;636 if (TargetIsArm != InstrIsBlx) {637 if (LLVM_LIKELY(TargetIsArm)) {638 // Change opcode BL -> BLX and fix range value: account for 4-byte639 // aligned destination while instruction may only be 2-byte aligned640 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;641 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitH;642 Value = alignTo(Value, 4);643 } else {644 // Change opcode BLX -> BL645 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;646 }647 }648 649 if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {650 if (!isInt<25>(Value))651 return makeTargetOutOfRangeError(G, B, E);652 writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2_J1J2(Value));653 } else {654 if (!isInt<22>(Value))655 return makeTargetOutOfRangeError(G, B, E);656 writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2(Value));657 }658 659 assert(((R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) ||660 (R.Lo & FixupInfo<Thumb_Call>::LoBitH) == 0) &&661 "Opcode BLX implies H bit is clear (avoid UB in BLX T2)");662 return Error::success();663 }664 665 case Thumb_MovwAbsNC: {666 uint16_t Value = (TargetAddress + Addend) & 0xffff;667 writeImmediate<Thumb_MovwAbsNC>(R, encodeImmMovtT1MovwT3(Value));668 return Error::success();669 }670 case Thumb_MovtAbs: {671 uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;672 writeImmediate<Thumb_MovtAbs>(R, encodeImmMovtT1MovwT3(Value));673 return Error::success();674 }675 case Thumb_MovwPrelNC: {676 uint16_t Value = ((TargetAddress + Addend - FixupAddress) & 0xffff);677 writeImmediate<Thumb_MovwPrelNC>(R, encodeImmMovtT1MovwT3(Value));678 return Error::success();679 }680 case Thumb_MovtPrel: {681 uint16_t Value = (((TargetAddress + Addend - FixupAddress) >> 16) & 0xffff);682 writeImmediate<Thumb_MovtPrel>(R, encodeImmMovtT1MovwT3(Value));683 return Error::success();684 }685 686 default:687 return make_error<JITLinkError>(688 "In graph " + G.getName() + ", section " + B.getSection().getName() +689 " encountered unfixable aarch32 edge kind " +690 G.getEdgeKindName(E.getKind()));691 }692}693 694const uint8_t GOTEntryInit[] = {695 0x00,696 0x00,697 0x00,698 0x00,699};700 701/// Create a new node in the link-graph for the given pointer value.702template <size_t Size>703static Block &allocPointer(LinkGraph &G, Section &S,704 const uint8_t (&Content)[Size]) {705 static_assert(Size == 4, "Pointers are 32-bit");706 constexpr uint64_t Alignment = 4;707 ArrayRef<char> Init(reinterpret_cast<const char *>(Content), Size);708 return G.createContentBlock(S, Init, orc::ExecutorAddr(), Alignment, 0);709}710 711Symbol &GOTBuilder::createEntry(LinkGraph &G, Symbol &Target) {712 if (!GOTSection)713 GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read);714 Block &B = allocPointer(G, *GOTSection, GOTEntryInit);715 constexpr int64_t GOTEntryAddend = 0;716 B.addEdge(Data_Pointer32, 0, Target, GOTEntryAddend);717 return G.addAnonymousSymbol(B, 0, B.getSize(), false, false);718}719 720bool GOTBuilder::visitEdge(LinkGraph &G, Block *B, Edge &E) {721 Edge::Kind KindToSet = Edge::Invalid;722 switch (E.getKind()) {723 case aarch32::Data_RequestGOTAndTransformToDelta32: {724 KindToSet = aarch32::Data_Delta32;725 break;726 }727 default:728 return false;729 }730 LLVM_DEBUG(dbgs() << " Transforming " << G.getEdgeKindName(E.getKind())731 << " edge at " << B->getFixupAddress(E) << " ("732 << B->getAddress() << " + "733 << formatv("{0:x}", E.getOffset()) << ") into "734 << G.getEdgeKindName(KindToSet) << "\n");735 E.setKind(KindToSet);736 E.setTarget(getEntryForTarget(G, E.getTarget()));737 return true;738}739 740const uint8_t ArmThumbv5LdrPc[] = {741 0x78, 0x47, // bx pc742 0xfd, 0xe7, // b #-6 ; Arm recommended sequence to follow bx pc743 0x04, 0xf0, 0x1f, 0xe5, // ldr pc, [pc,#-4] ; L1744 0x00, 0x00, 0x00, 0x00, // L1: .word S745};746 747const uint8_t Armv7ABS[] = {748 0x00, 0xc0, 0x00, 0xe3, // movw r12, #0x0000 ; lower 16-bit749 0x00, 0xc0, 0x40, 0xe3, // movt r12, #0x0000 ; upper 16-bit750 0x1c, 0xff, 0x2f, 0xe1 // bx r12751};752 753const uint8_t Thumbv7ABS[] = {754 0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000 ; lower 16-bit755 0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000 ; upper 16-bit756 0x60, 0x47 // bx r12757};758 759/// Create a new node in the link-graph for the given stub template.760template <size_t Size>761static Block &allocStub(LinkGraph &G, Section &S, const uint8_t (&Code)[Size]) {762 constexpr uint64_t Alignment = 4;763 ArrayRef<char> Template(reinterpret_cast<const char *>(Code), Size);764 return G.createContentBlock(S, Template, orc::ExecutorAddr(), Alignment, 0);765}766 767static Block &createStubPrev7(LinkGraph &G, Section &S, Symbol &Target) {768 Block &B = allocStub(G, S, ArmThumbv5LdrPc);769 B.addEdge(Data_Pointer32, 8, Target, 0);770 return B;771}772 773static Block &createStubThumbv7(LinkGraph &G, Section &S, Symbol &Target) {774 Block &B = allocStub(G, S, Thumbv7ABS);775 B.addEdge(Thumb_MovwAbsNC, 0, Target, 0);776 B.addEdge(Thumb_MovtAbs, 4, Target, 0);777 778 [[maybe_unused]] const char *StubPtr = B.getContent().data();779 [[maybe_unused]] HalfWords Reg12 = encodeRegMovtT1MovwT3(12);780 assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) &&781 checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) &&782 "Linker generated stubs may only corrupt register r12 (IP)");783 return B;784}785 786static Block &createStubArmv7(LinkGraph &G, Section &S, Symbol &Target) {787 Block &B = allocStub(G, S, Armv7ABS);788 B.addEdge(Arm_MovwAbsNC, 0, Target, 0);789 B.addEdge(Arm_MovtAbs, 4, Target, 0);790 791 [[maybe_unused]] const char *StubPtr = B.getContent().data();792 [[maybe_unused]] uint32_t Reg12 = encodeRegMovtA1MovwA2(12);793 assert(checkRegister<Arm_MovwAbsNC>(StubPtr, Reg12) &&794 checkRegister<Arm_MovtAbs>(StubPtr + 4, Reg12) &&795 "Linker generated stubs may only corrupt register r12 (IP)");796 return B;797}798 799static bool needsStub(const Edge &E) {800 Symbol &Target = E.getTarget();801 802 // Create stubs for external branch targets.803 if (!Target.isDefined()) {804 switch (E.getKind()) {805 case Arm_Call:806 case Arm_Jump24:807 case Thumb_Call:808 case Thumb_Jump24:809 return true;810 default:811 return false;812 }813 }814 815 // For local targets, create interworking stubs if we switch Arm/Thumb with an816 // instruction that cannot switch the instruction set state natively.817 bool TargetIsThumb = Target.getTargetFlags() & ThumbSymbol;818 switch (E.getKind()) {819 case Arm_Jump24:820 return TargetIsThumb; // Branch to Thumb needs interworking stub821 case Thumb_Jump24:822 return !TargetIsThumb; // Branch to Arm needs interworking stub823 default:824 break;825 }826 827 return false;828}829 830// The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only831// for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm832// entrypoint at offset 4. Arm branches always use that one.833Symbol *StubsManager_prev7::getOrCreateSlotEntrypoint(LinkGraph &G,834 StubMapEntry &Slot,835 bool Thumb) {836 constexpr orc::ExecutorAddrDiff ThumbEntrypointOffset = 0;837 constexpr orc::ExecutorAddrDiff ArmEntrypointOffset = 4;838 if (Thumb && !Slot.ThumbEntry) {839 Slot.ThumbEntry =840 &G.addAnonymousSymbol(*Slot.B, ThumbEntrypointOffset, 4, true, false);841 Slot.ThumbEntry->setTargetFlags(ThumbSymbol);842 }843 if (!Thumb && !Slot.ArmEntry)844 Slot.ArmEntry =845 &G.addAnonymousSymbol(*Slot.B, ArmEntrypointOffset, 8, true, false);846 return Thumb ? Slot.ThumbEntry : Slot.ArmEntry;847}848 849bool StubsManager_prev7::visitEdge(LinkGraph &G, Block *B, Edge &E) {850 if (!needsStub(E))851 return false;852 853 Symbol &Target = E.getTarget();854 assert(Target.hasName() && "Edge cannot point to anonymous target");855 auto [Slot, NewStub] = getStubMapSlot(*Target.getName());856 857 if (NewStub) {858 if (!StubsSection)859 StubsSection = &G.createSection(getSectionName(),860 orc::MemProt::Read | orc::MemProt::Exec);861 LLVM_DEBUG({862 dbgs() << " Created stub entry for " << Target.getName() << " in "863 << StubsSection->getName() << "\n";864 });865 Slot->B = &createStubPrev7(G, *StubsSection, Target);866 }867 868 // The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only869 // for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm870 // entrypoint at offset 4. Arm branches always use that one.871 bool UseThumb = E.getKind() == Thumb_Jump24;872 Symbol *StubEntrypoint = getOrCreateSlotEntrypoint(G, *Slot, UseThumb);873 874 LLVM_DEBUG({875 dbgs() << " Using " << (UseThumb ? "Thumb" : "Arm") << " entrypoint "876 << *StubEntrypoint << " in "877 << StubEntrypoint->getSection().getName() << "\n";878 });879 880 E.setTarget(*StubEntrypoint);881 return true;882}883 884bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {885 if (!needsStub(E))886 return false;887 888 // Stub Arm/Thumb follows instruction set state at relocation site.889 // TODO: We may reduce them at relaxation time and reuse freed slots.890 bool MakeThumb = (E.getKind() > LastArmRelocation);891 LLVM_DEBUG(dbgs() << " Preparing " << (MakeThumb ? "Thumb" : "Arm")892 << " stub for " << G.getEdgeKindName(E.getKind())893 << " edge at " << B->getFixupAddress(E) << " ("894 << B->getAddress() << " + "895 << formatv("{0:x}", E.getOffset()) << ")\n");896 897 Symbol &Target = E.getTarget();898 assert(Target.hasName() && "Edge cannot point to anonymous target");899 Symbol *&StubSymbol = getStubSymbolSlot(*Target.getName(), MakeThumb);900 901 if (!StubSymbol) {902 if (!StubsSection)903 StubsSection = &G.createSection(getSectionName(),904 orc::MemProt::Read | orc::MemProt::Exec);905 Block &B = MakeThumb ? createStubThumbv7(G, *StubsSection, Target)906 : createStubArmv7(G, *StubsSection, Target);907 StubSymbol = &G.addAnonymousSymbol(B, 0, B.getSize(), true, false);908 if (MakeThumb)909 StubSymbol->setTargetFlags(ThumbSymbol);910 911 LLVM_DEBUG({912 dbgs() << " Created " << (MakeThumb ? "Thumb" : "Arm") << " entry for "913 << Target.getName() << " in " << StubsSection->getName() << ": "914 << *StubSymbol << "\n";915 });916 }917 918 assert(MakeThumb == (StubSymbol->getTargetFlags() & ThumbSymbol) &&919 "Instruction set states of stub and relocation site should be equal");920 LLVM_DEBUG({921 dbgs() << " Using " << (MakeThumb ? "Thumb" : "Arm") << " entry "922 << *StubSymbol << " in " << StubSymbol->getSection().getName()923 << "\n";924 });925 926 E.setTarget(*StubSymbol);927 return true;928}929 930const char *getEdgeKindName(Edge::Kind K) {931#define KIND_NAME_CASE(K) \932 case K: \933 return #K;934 935 switch (K) {936 KIND_NAME_CASE(Data_Delta32)937 KIND_NAME_CASE(Data_Pointer32)938 KIND_NAME_CASE(Data_PRel31)939 KIND_NAME_CASE(Data_RequestGOTAndTransformToDelta32)940 KIND_NAME_CASE(Arm_Call)941 KIND_NAME_CASE(Arm_Jump24)942 KIND_NAME_CASE(Arm_MovwAbsNC)943 KIND_NAME_CASE(Arm_MovtAbs)944 KIND_NAME_CASE(Thumb_Call)945 KIND_NAME_CASE(Thumb_Jump24)946 KIND_NAME_CASE(Thumb_MovwAbsNC)947 KIND_NAME_CASE(Thumb_MovtAbs)948 KIND_NAME_CASE(Thumb_MovwPrelNC)949 KIND_NAME_CASE(Thumb_MovtPrel)950 KIND_NAME_CASE(None)951 default:952 return getGenericEdgeKindName(K);953 }954#undef KIND_NAME_CASE955}956 957const char *getCPUArchName(ARMBuildAttrs::CPUArch K) {958#define CPUARCH_NAME_CASE(K) \959 case K: \960 return #K;961 962 using namespace ARMBuildAttrs;963 switch (K) {964 CPUARCH_NAME_CASE(Pre_v4)965 CPUARCH_NAME_CASE(v4)966 CPUARCH_NAME_CASE(v4T)967 CPUARCH_NAME_CASE(v5T)968 CPUARCH_NAME_CASE(v5TE)969 CPUARCH_NAME_CASE(v5TEJ)970 CPUARCH_NAME_CASE(v6)971 CPUARCH_NAME_CASE(v6KZ)972 CPUARCH_NAME_CASE(v6T2)973 CPUARCH_NAME_CASE(v6K)974 CPUARCH_NAME_CASE(v7)975 CPUARCH_NAME_CASE(v6_M)976 CPUARCH_NAME_CASE(v6S_M)977 CPUARCH_NAME_CASE(v7E_M)978 CPUARCH_NAME_CASE(v8_A)979 CPUARCH_NAME_CASE(v8_R)980 CPUARCH_NAME_CASE(v8_M_Base)981 CPUARCH_NAME_CASE(v8_M_Main)982 CPUARCH_NAME_CASE(v8_1_M_Main)983 CPUARCH_NAME_CASE(v9_A)984 }985 llvm_unreachable("Missing CPUArch in switch?");986#undef CPUARCH_NAME_CASE987}988 989} // namespace aarch32990} // namespace jitlink991} // namespace llvm992