1569 lines · cpp
1//===- RISCV.cpp ----------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "InputFiles.h"10#include "OutputSections.h"11#include "RISCVInternalRelocations.h"12#include "RelocScan.h"13#include "Symbols.h"14#include "SyntheticSections.h"15#include "Target.h"16#include "llvm/Support/ELFAttributes.h"17#include "llvm/Support/LEB128.h"18#include "llvm/Support/RISCVAttributeParser.h"19#include "llvm/Support/RISCVAttributes.h"20#include "llvm/Support/TimeProfiler.h"21#include "llvm/TargetParser/RISCVISAInfo.h"22 23using namespace llvm;24using namespace llvm::object;25using namespace llvm::support::endian;26using namespace llvm::ELF;27using namespace lld;28using namespace lld::elf;29 30namespace {31 32class RISCV final : public TargetInfo {33public:34 RISCV(Ctx &);35 uint32_t calcEFlags() const override;36 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;37 void writeGotHeader(uint8_t *buf) const override;38 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;39 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;40 void writePltHeader(uint8_t *buf) const override;41 void writePlt(uint8_t *buf, const Symbol &sym,42 uint64_t pltEntryAddr) const override;43 template <class ELFT, class RelTy>44 void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);45 template <class ELFT> void scanSection1(InputSectionBase &);46 void scanSection(InputSectionBase &) override;47 RelType getDynRel(RelType type) const override;48 RelExpr getRelExpr(RelType type, const Symbol &s,49 const uint8_t *loc) const override;50 void relocate(uint8_t *loc, const Relocation &rel,51 uint64_t val) const override;52 void relocateAlloc(InputSection &sec, uint8_t *buf) const override;53 bool relaxOnce(int pass) const override;54 template <class ELFT, class RelTy>55 bool synthesizeAlignForInput(uint64_t &dot, InputSection *sec,56 Relocs<RelTy> rels);57 template <class ELFT, class RelTy>58 void finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,59 Relocs<RelTy> rels);60 template <class ELFT>61 bool synthesizeAlignAux(uint64_t &dot, InputSection *sec);62 bool synthesizeAlign(uint64_t &dot, InputSection *sec) override;63 void finalizeRelax(int passes) const override;64 65 // The following two variables are used by synthesized ALIGN relocations.66 InputSection *baseSec = nullptr;67 // r_offset and r_addend pairs.68 SmallVector<std::pair<uint64_t, uint64_t>, 0> synthesizedAligns;69};70 71} // end anonymous namespace72 73// These are internal relocation numbers for GP/X0 relaxation. They aren't part74// of the psABI spec.75#define INTERNAL_R_RISCV_GPREL_I 25676#define INTERNAL_R_RISCV_GPREL_S 25777#define INTERNAL_R_RISCV_X0REL_I 25878#define INTERNAL_R_RISCV_X0REL_S 25979 80const uint64_t dtpOffset = 0x800;81 82namespace {83enum Op {84 ADDI = 0x13,85 AUIPC = 0x17,86 JALR = 0x67,87 LD = 0x3003,88 LUI = 0x37,89 LW = 0x2003,90 SRLI = 0x5013,91 SUB = 0x40000033,92};93 94enum Reg {95 X_X0 = 0,96 X_RA = 1,97 X_GP = 3,98 X_TP = 4,99 X_T0 = 5,100 X_T1 = 6,101 X_T2 = 7,102 X_A0 = 10,103 X_T3 = 28,104};105} // namespace106 107static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; }108static uint32_t lo12(uint32_t val) { return val & 4095; }109 110static uint32_t itype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t imm) {111 return op | (rd << 7) | (rs1 << 15) | (imm << 20);112}113static uint32_t rtype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t rs2) {114 return op | (rd << 7) | (rs1 << 15) | (rs2 << 20);115}116static uint32_t utype(uint32_t op, uint32_t rd, uint32_t imm) {117 return op | (rd << 7) | (imm << 12);118}119 120// Extract bits v[begin:end], where range is inclusive, and begin must be < 63.121static uint32_t extractBits(uint64_t v, uint32_t begin, uint32_t end) {122 return (v & ((1ULL << (begin + 1)) - 1)) >> end;123}124 125static uint32_t setLO12_I(uint32_t insn, uint32_t imm) {126 return (insn & 0xfffff) | (imm << 20);127}128static uint32_t setLO12_S(uint32_t insn, uint32_t imm) {129 return (insn & 0x1fff07f) | (extractBits(imm, 11, 5) << 25) |130 (extractBits(imm, 4, 0) << 7);131}132 133RISCV::RISCV(Ctx &ctx) : TargetInfo(ctx) {134 copyRel = R_RISCV_COPY;135 pltRel = R_RISCV_JUMP_SLOT;136 relativeRel = R_RISCV_RELATIVE;137 iRelativeRel = R_RISCV_IRELATIVE;138 if (ctx.arg.is64) {139 symbolicRel = R_RISCV_64;140 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD64;141 tlsOffsetRel = R_RISCV_TLS_DTPREL64;142 tlsGotRel = R_RISCV_TLS_TPREL64;143 } else {144 symbolicRel = R_RISCV_32;145 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD32;146 tlsOffsetRel = R_RISCV_TLS_DTPREL32;147 tlsGotRel = R_RISCV_TLS_TPREL32;148 }149 gotRel = symbolicRel;150 tlsDescRel = R_RISCV_TLSDESC;151 152 // .got[0] = _DYNAMIC153 gotHeaderEntriesNum = 1;154 155 // .got.plt[0] = _dl_runtime_resolve, .got.plt[1] = link_map156 gotPltHeaderEntriesNum = 2;157 158 pltHeaderSize = 32;159 pltEntrySize = 16;160 ipltEntrySize = 16;161}162 163static uint32_t getEFlags(Ctx &ctx, InputFile *f) {164 if (ctx.arg.is64)165 return cast<ObjFile<ELF64LE>>(f)->getObj().getHeader().e_flags;166 return cast<ObjFile<ELF32LE>>(f)->getObj().getHeader().e_flags;167}168 169uint32_t RISCV::calcEFlags() const {170 // If there are only binary input files (from -b binary), use a171 // value of 0 for the ELF header flags.172 if (ctx.objectFiles.empty())173 return 0;174 175 uint32_t target = getEFlags(ctx, ctx.objectFiles.front());176 for (InputFile *f : ctx.objectFiles) {177 uint32_t eflags = getEFlags(ctx, f);178 if (eflags & EF_RISCV_RVC)179 target |= EF_RISCV_RVC;180 181 if ((eflags & EF_RISCV_FLOAT_ABI) != (target & EF_RISCV_FLOAT_ABI))182 Err(ctx) << f183 << ": cannot link object files with different "184 "floating-point ABI from "185 << ctx.objectFiles[0];186 187 if ((eflags & EF_RISCV_RVE) != (target & EF_RISCV_RVE))188 Err(ctx) << f << ": cannot link object files with different EF_RISCV_RVE";189 }190 191 return target;192}193 194int64_t RISCV::getImplicitAddend(const uint8_t *buf, RelType type) const {195 switch (type) {196 default:197 InternalErr(ctx, buf) << "cannot read addend for relocation " << type;198 return 0;199 case R_RISCV_32:200 case R_RISCV_TLS_DTPMOD32:201 case R_RISCV_TLS_DTPREL32:202 case R_RISCV_TLS_TPREL32:203 return SignExtend64<32>(read32le(buf));204 case R_RISCV_64:205 case R_RISCV_TLS_DTPMOD64:206 case R_RISCV_TLS_DTPREL64:207 case R_RISCV_TLS_TPREL64:208 return read64le(buf);209 case R_RISCV_RELATIVE:210 case R_RISCV_IRELATIVE:211 return ctx.arg.is64 ? read64le(buf) : read32le(buf);212 case R_RISCV_NONE:213 case R_RISCV_JUMP_SLOT:214 // These relocations are defined as not having an implicit addend.215 return 0;216 case R_RISCV_TLSDESC:217 return ctx.arg.is64 ? read64le(buf + 8) : read32le(buf + 4);218 }219}220 221void RISCV::writeGotHeader(uint8_t *buf) const {222 if (ctx.arg.is64)223 write64le(buf, ctx.mainPart->dynamic->getVA());224 else225 write32le(buf, ctx.mainPart->dynamic->getVA());226}227 228void RISCV::writeGotPlt(uint8_t *buf, const Symbol &s) const {229 if (ctx.arg.is64)230 write64le(buf, ctx.in.plt->getVA());231 else232 write32le(buf, ctx.in.plt->getVA());233}234 235void RISCV::writeIgotPlt(uint8_t *buf, const Symbol &s) const {236 if (ctx.arg.writeAddends) {237 if (ctx.arg.is64)238 write64le(buf, s.getVA(ctx));239 else240 write32le(buf, s.getVA(ctx));241 }242}243 244void RISCV::writePltHeader(uint8_t *buf) const {245 // 1: auipc t2, %pcrel_hi(.got.plt)246 // sub t1, t1, t3247 // l[wd] t3, %pcrel_lo(1b)(t2); t3 = _dl_runtime_resolve248 // addi t1, t1, -pltHeaderSize-12; t1 = &.plt[i] - &.plt[0]249 // addi t0, t2, %pcrel_lo(1b)250 // srli t1, t1, (rv64?1:2); t1 = &.got.plt[i] - &.got.plt[0]251 // l[wd] t0, Wordsize(t0); t0 = link_map252 // jr t3253 uint32_t offset = ctx.in.gotPlt->getVA() - ctx.in.plt->getVA();254 uint32_t load = ctx.arg.is64 ? LD : LW;255 write32le(buf + 0, utype(AUIPC, X_T2, hi20(offset)));256 write32le(buf + 4, rtype(SUB, X_T1, X_T1, X_T3));257 write32le(buf + 8, itype(load, X_T3, X_T2, lo12(offset)));258 write32le(buf + 12, itype(ADDI, X_T1, X_T1, -ctx.target->pltHeaderSize - 12));259 write32le(buf + 16, itype(ADDI, X_T0, X_T2, lo12(offset)));260 write32le(buf + 20, itype(SRLI, X_T1, X_T1, ctx.arg.is64 ? 1 : 2));261 write32le(buf + 24, itype(load, X_T0, X_T0, ctx.arg.wordsize));262 write32le(buf + 28, itype(JALR, 0, X_T3, 0));263}264 265void RISCV::writePlt(uint8_t *buf, const Symbol &sym,266 uint64_t pltEntryAddr) const {267 // 1: auipc t3, %pcrel_hi(f@.got.plt)268 // l[wd] t3, %pcrel_lo(1b)(t3)269 // jalr t1, t3270 // nop271 uint32_t offset = sym.getGotPltVA(ctx) - pltEntryAddr;272 write32le(buf + 0, utype(AUIPC, X_T3, hi20(offset)));273 write32le(buf + 4, itype(ctx.arg.is64 ? LD : LW, X_T3, X_T3, lo12(offset)));274 write32le(buf + 8, itype(JALR, X_T1, X_T3, 0));275 write32le(buf + 12, itype(ADDI, 0, 0, 0));276}277 278RelType RISCV::getDynRel(RelType type) const {279 return type == ctx.target->symbolicRel ? type280 : static_cast<RelType>(R_RISCV_NONE);281}282 283RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,284 const uint8_t *loc) const {285 switch (type) {286 case R_RISCV_NONE:287 case R_RISCV_VENDOR:288 return R_NONE;289 case R_RISCV_32:290 case R_RISCV_64:291 case R_RISCV_HI20:292 case R_RISCV_LO12_I:293 case R_RISCV_LO12_S:294 return R_ABS;295 case R_RISCV_ADD8:296 case R_RISCV_ADD16:297 case R_RISCV_ADD32:298 case R_RISCV_ADD64:299 case R_RISCV_SET6:300 case R_RISCV_SET8:301 case R_RISCV_SET16:302 case R_RISCV_SET32:303 case R_RISCV_SUB6:304 case R_RISCV_SUB8:305 case R_RISCV_SUB16:306 case R_RISCV_SUB32:307 case R_RISCV_SUB64:308 return RE_RISCV_ADD;309 case R_RISCV_JAL:310 case R_RISCV_BRANCH:311 case R_RISCV_PCREL_HI20:312 case R_RISCV_RVC_BRANCH:313 case R_RISCV_RVC_JUMP:314 case R_RISCV_32_PCREL:315 return R_PC;316 case R_RISCV_CALL:317 case R_RISCV_CALL_PLT:318 case R_RISCV_PLT32:319 return R_PLT_PC;320 case R_RISCV_GOT_HI20:321 case R_RISCV_GOT32_PCREL:322 return R_GOT_PC;323 case R_RISCV_PCREL_LO12_I:324 case R_RISCV_PCREL_LO12_S:325 return RE_RISCV_PC_INDIRECT;326 case R_RISCV_TLSDESC_HI20:327 case R_RISCV_TLSDESC_LOAD_LO12:328 case R_RISCV_TLSDESC_ADD_LO12:329 return R_TLSDESC_PC;330 case R_RISCV_TLSDESC_CALL:331 return R_TLSDESC_CALL;332 case R_RISCV_TLS_GD_HI20:333 return R_TLSGD_PC;334 case R_RISCV_TLS_GOT_HI20:335 return R_GOT_PC;336 case R_RISCV_TPREL_HI20:337 case R_RISCV_TPREL_LO12_I:338 case R_RISCV_TPREL_LO12_S:339 return R_TPREL;340 case R_RISCV_ALIGN:341 return R_RELAX_HINT;342 case R_RISCV_TPREL_ADD:343 case R_RISCV_RELAX:344 return ctx.arg.relax ? R_RELAX_HINT : R_NONE;345 case R_RISCV_SET_ULEB128:346 case R_RISCV_SUB_ULEB128:347 return RE_RISCV_LEB128;348 default:349 if (type.v & INTERNAL_RISCV_VENDOR_MASK) {350 Err(ctx) << getErrorLoc(ctx, loc)351 << "unsupported vendor-specific relocation " << type352 << " against symbol " << &s;353 return R_NONE;354 }355 Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation ("356 << (type.v & ~INTERNAL_RISCV_VENDOR_MASK) << ") against symbol "357 << &s;358 return R_NONE;359 }360}361 362void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {363 const unsigned bits = ctx.arg.wordsize * 8;364 365 switch (rel.type) {366 case R_RISCV_32:367 write32le(loc, val);368 return;369 case R_RISCV_64:370 write64le(loc, val);371 return;372 373 case R_RISCV_RVC_BRANCH: {374 checkInt(ctx, loc, val, 9, rel);375 checkAlignment(ctx, loc, val, 2, rel);376 uint16_t insn = read16le(loc) & 0xE383;377 uint16_t imm8 = extractBits(val, 8, 8) << 12;378 uint16_t imm4_3 = extractBits(val, 4, 3) << 10;379 uint16_t imm7_6 = extractBits(val, 7, 6) << 5;380 uint16_t imm2_1 = extractBits(val, 2, 1) << 3;381 uint16_t imm5 = extractBits(val, 5, 5) << 2;382 insn |= imm8 | imm4_3 | imm7_6 | imm2_1 | imm5;383 384 write16le(loc, insn);385 return;386 }387 388 case R_RISCV_RVC_JUMP: {389 checkInt(ctx, loc, val, 12, rel);390 checkAlignment(ctx, loc, val, 2, rel);391 uint16_t insn = read16le(loc) & 0xE003;392 uint16_t imm11 = extractBits(val, 11, 11) << 12;393 uint16_t imm4 = extractBits(val, 4, 4) << 11;394 uint16_t imm9_8 = extractBits(val, 9, 8) << 9;395 uint16_t imm10 = extractBits(val, 10, 10) << 8;396 uint16_t imm6 = extractBits(val, 6, 6) << 7;397 uint16_t imm7 = extractBits(val, 7, 7) << 6;398 uint16_t imm3_1 = extractBits(val, 3, 1) << 3;399 uint16_t imm5 = extractBits(val, 5, 5) << 2;400 insn |= imm11 | imm4 | imm9_8 | imm10 | imm6 | imm7 | imm3_1 | imm5;401 402 write16le(loc, insn);403 return;404 }405 406 case R_RISCV_JAL: {407 checkInt(ctx, loc, val, 21, rel);408 checkAlignment(ctx, loc, val, 2, rel);409 410 uint32_t insn = read32le(loc) & 0xFFF;411 uint32_t imm20 = extractBits(val, 20, 20) << 31;412 uint32_t imm10_1 = extractBits(val, 10, 1) << 21;413 uint32_t imm11 = extractBits(val, 11, 11) << 20;414 uint32_t imm19_12 = extractBits(val, 19, 12) << 12;415 insn |= imm20 | imm10_1 | imm11 | imm19_12;416 417 write32le(loc, insn);418 return;419 }420 421 case R_RISCV_BRANCH: {422 checkInt(ctx, loc, val, 13, rel);423 checkAlignment(ctx, loc, val, 2, rel);424 425 uint32_t insn = read32le(loc) & 0x1FFF07F;426 uint32_t imm12 = extractBits(val, 12, 12) << 31;427 uint32_t imm10_5 = extractBits(val, 10, 5) << 25;428 uint32_t imm4_1 = extractBits(val, 4, 1) << 8;429 uint32_t imm11 = extractBits(val, 11, 11) << 7;430 insn |= imm12 | imm10_5 | imm4_1 | imm11;431 432 write32le(loc, insn);433 return;434 }435 436 // auipc + jalr pair437 case R_RISCV_CALL:438 case R_RISCV_CALL_PLT: {439 int64_t hi = SignExtend64(val + 0x800, bits) >> 12;440 checkInt(ctx, loc, hi, 20, rel);441 if (isInt<20>(hi)) {442 relocateNoSym(loc, R_RISCV_PCREL_HI20, val);443 relocateNoSym(loc + 4, R_RISCV_PCREL_LO12_I, val);444 }445 return;446 }447 448 case R_RISCV_GOT_HI20:449 case R_RISCV_PCREL_HI20:450 case R_RISCV_TLSDESC_HI20:451 case R_RISCV_TLS_GD_HI20:452 case R_RISCV_TLS_GOT_HI20:453 case R_RISCV_TPREL_HI20:454 case R_RISCV_HI20: {455 uint64_t hi = val + 0x800;456 checkInt(ctx, loc, SignExtend64(hi, bits) >> 12, 20, rel);457 write32le(loc, (read32le(loc) & 0xFFF) | (hi & 0xFFFFF000));458 return;459 }460 461 case R_RISCV_PCREL_LO12_I:462 case R_RISCV_TLSDESC_LOAD_LO12:463 case R_RISCV_TLSDESC_ADD_LO12:464 case R_RISCV_TPREL_LO12_I:465 case R_RISCV_LO12_I: {466 uint64_t hi = (val + 0x800) >> 12;467 uint64_t lo = val - (hi << 12);468 write32le(loc, setLO12_I(read32le(loc), lo & 0xfff));469 return;470 }471 472 case R_RISCV_PCREL_LO12_S:473 case R_RISCV_TPREL_LO12_S:474 case R_RISCV_LO12_S: {475 uint64_t hi = (val + 0x800) >> 12;476 uint64_t lo = val - (hi << 12);477 write32le(loc, setLO12_S(read32le(loc), lo));478 return;479 }480 481 case INTERNAL_R_RISCV_X0REL_I:482 case INTERNAL_R_RISCV_X0REL_S: {483 checkInt(ctx, loc, val, 12, rel);484 uint32_t insn = (read32le(loc) & ~(31 << 15)) | (X_X0 << 15);485 if (rel.type == INTERNAL_R_RISCV_X0REL_I)486 insn = setLO12_I(insn, val);487 else488 insn = setLO12_S(insn, val);489 write32le(loc, insn);490 return;491 }492 493 case INTERNAL_R_RISCV_GPREL_I:494 case INTERNAL_R_RISCV_GPREL_S: {495 Defined *gp = ctx.sym.riscvGlobalPointer;496 int64_t displace = SignExtend64(val - gp->getVA(ctx), bits);497 checkInt(ctx, loc, displace, 12, rel);498 uint32_t insn = (read32le(loc) & ~(31 << 15)) | (X_GP << 15);499 if (rel.type == INTERNAL_R_RISCV_GPREL_I)500 insn = setLO12_I(insn, displace);501 else502 insn = setLO12_S(insn, displace);503 write32le(loc, insn);504 return;505 }506 507 case R_RISCV_ADD8:508 *loc += val;509 return;510 case R_RISCV_ADD16:511 write16le(loc, read16le(loc) + val);512 return;513 case R_RISCV_ADD32:514 write32le(loc, read32le(loc) + val);515 return;516 case R_RISCV_ADD64:517 write64le(loc, read64le(loc) + val);518 return;519 case R_RISCV_SUB6:520 *loc = (*loc & 0xc0) | (((*loc & 0x3f) - val) & 0x3f);521 return;522 case R_RISCV_SUB8:523 *loc -= val;524 return;525 case R_RISCV_SUB16:526 write16le(loc, read16le(loc) - val);527 return;528 case R_RISCV_SUB32:529 write32le(loc, read32le(loc) - val);530 return;531 case R_RISCV_SUB64:532 write64le(loc, read64le(loc) - val);533 return;534 case R_RISCV_SET6:535 *loc = (*loc & 0xc0) | (val & 0x3f);536 return;537 case R_RISCV_SET8:538 *loc = val;539 return;540 case R_RISCV_SET16:541 write16le(loc, val);542 return;543 case R_RISCV_SET32:544 case R_RISCV_32_PCREL:545 case R_RISCV_PLT32:546 case R_RISCV_GOT32_PCREL:547 checkInt(ctx, loc, val, 32, rel);548 write32le(loc, val);549 return;550 551 case R_RISCV_TLS_DTPREL32:552 write32le(loc, val - dtpOffset);553 break;554 case R_RISCV_TLS_DTPREL64:555 write64le(loc, val - dtpOffset);556 break;557 558 case R_RISCV_RELAX:559 return;560 case R_RISCV_TLSDESC:561 // The addend is stored in the second word.562 if (ctx.arg.is64)563 write64le(loc + 8, val);564 else565 write32le(loc + 4, val);566 break;567 default:568 llvm_unreachable("unknown relocation");569 }570}571 572static bool relaxable(ArrayRef<Relocation> relocs, size_t i) {573 return i + 1 != relocs.size() && relocs[i + 1].type == R_RISCV_RELAX;574}575 576static void tlsdescToIe(Ctx &ctx, uint8_t *loc, const Relocation &rel,577 uint64_t val) {578 switch (rel.type) {579 case R_RISCV_TLSDESC_HI20:580 case R_RISCV_TLSDESC_LOAD_LO12:581 write32le(loc, 0x00000013); // nop582 break;583 case R_RISCV_TLSDESC_ADD_LO12:584 write32le(loc, utype(AUIPC, X_A0, hi20(val))); // auipc a0,<hi20>585 break;586 case R_RISCV_TLSDESC_CALL:587 if (ctx.arg.is64)588 write32le(loc, itype(LD, X_A0, X_A0, lo12(val))); // ld a0,<lo12>(a0)589 else590 write32le(loc, itype(LW, X_A0, X_A0, lo12(val))); // lw a0,<lo12>(a0)591 break;592 default:593 llvm_unreachable("unsupported relocation for TLSDESC to IE");594 }595}596 597static void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) {598 switch (rel.type) {599 case R_RISCV_TLSDESC_HI20:600 case R_RISCV_TLSDESC_LOAD_LO12:601 write32le(loc, 0x00000013); // nop602 return;603 case R_RISCV_TLSDESC_ADD_LO12:604 if (isInt<12>(val))605 write32le(loc, 0x00000013); // nop606 else607 write32le(loc, utype(LUI, X_A0, hi20(val))); // lui a0,<hi20>608 return;609 case R_RISCV_TLSDESC_CALL:610 if (isInt<12>(val))611 write32le(loc, itype(ADDI, X_A0, 0, val)); // addi a0,zero,<lo12>612 else613 write32le(loc, itype(ADDI, X_A0, X_A0, lo12(val))); // addi a0,a0,<lo12>614 return;615 default:616 llvm_unreachable("unsupported relocation for TLSDESC to LE");617 }618}619 620void RISCV::relocateAlloc(InputSection &sec, uint8_t *buf) const {621 uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff;622 uint64_t tlsdescVal = 0;623 bool tlsdescRelax = false, isToLe = false;624 const ArrayRef<Relocation> relocs = sec.relocs();625 for (size_t i = 0, size = relocs.size(); i != size; ++i) {626 const Relocation &rel = relocs[i];627 uint8_t *loc = buf + rel.offset;628 uint64_t val = sec.getRelocTargetVA(ctx, rel, secAddr + rel.offset);629 630 switch (rel.expr) {631 case R_RELAX_HINT:632 continue;633 case R_TLSDESC_PC:634 // For R_RISCV_TLSDESC_HI20, store &got(sym)-PC to be used by the635 // following two instructions L[DW] and ADDI.636 if (rel.type == R_RISCV_TLSDESC_HI20)637 tlsdescVal = val;638 else639 val = tlsdescVal;640 break;641 case R_RELAX_TLS_GD_TO_IE:642 // Only R_RISCV_TLSDESC_HI20 reaches here. tlsdescVal will be finalized643 // after we see R_RISCV_TLSDESC_ADD_LO12 in the R_RELAX_TLS_GD_TO_LE case.644 // The net effect is that tlsdescVal will be smaller than `val` to take645 // into account of NOP instructions (in the absence of R_RISCV_RELAX)646 // before AUIPC.647 tlsdescVal = val + rel.offset;648 isToLe = false;649 tlsdescRelax = relaxable(relocs, i);650 if (!tlsdescRelax)651 tlsdescToIe(ctx, loc, rel, val);652 continue;653 case R_RELAX_TLS_GD_TO_LE:654 // See the comment in handleTlsRelocation. For TLSDESC=>IE,655 // R_RISCV_TLSDESC_{LOAD_LO12,ADD_LO12,CALL} also reach here. If isToLe is656 // false, this is actually TLSDESC=>IE optimization.657 if (rel.type == R_RISCV_TLSDESC_HI20) {658 tlsdescVal = val;659 isToLe = true;660 tlsdescRelax = relaxable(relocs, i);661 } else {662 if (!isToLe && rel.type == R_RISCV_TLSDESC_ADD_LO12)663 tlsdescVal -= rel.offset;664 val = tlsdescVal;665 }666 // When NOP conversion is eligible and relaxation applies, don't write a667 // NOP in case an unrelated instruction follows the current instruction.668 if (tlsdescRelax &&669 (rel.type == R_RISCV_TLSDESC_HI20 ||670 rel.type == R_RISCV_TLSDESC_LOAD_LO12 ||671 (rel.type == R_RISCV_TLSDESC_ADD_LO12 && isToLe && !hi20(val))))672 continue;673 if (isToLe)674 tlsdescToLe(loc, rel, val);675 else676 tlsdescToIe(ctx, loc, rel, val);677 continue;678 case RE_RISCV_LEB128:679 if (i + 1 < size) {680 const Relocation &rel1 = relocs[i + 1];681 if (rel.type == R_RISCV_SET_ULEB128 &&682 rel1.type == R_RISCV_SUB_ULEB128 && rel.offset == rel1.offset) {683 auto val = rel.sym->getVA(ctx, rel.addend) -684 rel1.sym->getVA(ctx, rel1.addend);685 if (overwriteULEB128(loc, val) >= 0x80)686 Err(ctx) << sec.getLocation(rel.offset) << ": ULEB128 value " << val687 << " exceeds available space; references '" << rel.sym688 << "'";689 ++i;690 continue;691 }692 }693 Err(ctx) << sec.getLocation(rel.offset)694 << ": R_RISCV_SET_ULEB128 not paired with R_RISCV_SUB_SET128";695 return;696 default:697 break;698 }699 relocate(loc, rel, val);700 }701}702 703void elf::initSymbolAnchors(Ctx &ctx) {704 SmallVector<InputSection *, 0> storage;705 for (OutputSection *osec : ctx.outputSections) {706 if (!(osec->flags & SHF_EXECINSTR))707 continue;708 for (InputSection *sec : getInputSections(*osec, storage)) {709 sec->relaxAux = make<RelaxAux>();710 if (sec->relocs().size()) {711 sec->relaxAux->relocDeltas =712 std::make_unique<uint32_t[]>(sec->relocs().size());713 sec->relaxAux->relocTypes =714 std::make_unique<RelType[]>(sec->relocs().size());715 }716 }717 }718 // Store anchors (st_value and st_value+st_size) for symbols relative to text719 // sections.720 //721 // For a defined symbol foo, we may have `d->file != file` with --wrap=foo.722 // We should process foo, as the defining object file's symbol table may not723 // contain foo after redirectSymbols changed the foo entry to __wrap_foo. To724 // avoid adding a Defined that is undefined in one object file, use725 // `!d->scriptDefined` to exclude symbols that are definitely not wrapped.726 //727 // `relaxAux->anchors` may contain duplicate symbols, but that is fine.728 for (InputFile *file : ctx.objectFiles)729 for (Symbol *sym : file->getSymbols()) {730 auto *d = dyn_cast<Defined>(sym);731 if (!d || (d->file != file && !d->scriptDefined))732 continue;733 if (auto *sec = dyn_cast_or_null<InputSection>(d->section))734 if (sec->flags & SHF_EXECINSTR && sec->relaxAux) {735 // If sec is discarded, relaxAux will be nullptr.736 sec->relaxAux->anchors.push_back({d->value, d, false});737 sec->relaxAux->anchors.push_back({d->value + d->size, d, true});738 }739 }740 // Sort anchors by offset so that we can find the closest relocation741 // efficiently. For a zero size symbol, ensure that its start anchor precedes742 // its end anchor. For two symbols with anchors at the same offset, their743 // order does not matter.744 for (OutputSection *osec : ctx.outputSections) {745 if (!(osec->flags & SHF_EXECINSTR))746 continue;747 for (InputSection *sec : getInputSections(*osec, storage)) {748 llvm::sort(sec->relaxAux->anchors, [](auto &a, auto &b) {749 return std::make_pair(a.offset, a.end) <750 std::make_pair(b.offset, b.end);751 });752 }753 }754}755 756// Relax R_RISCV_CALL/R_RISCV_CALL_PLT auipc+jalr to c.j, c.jal, or jal.757static void relaxCall(Ctx &ctx, const InputSection &sec, size_t i, uint64_t loc,758 Relocation &r, uint32_t &remove) {759 const bool rvc = getEFlags(ctx, sec.file) & EF_RISCV_RVC;760 const Symbol &sym = *r.sym;761 const uint64_t insnPair = read64le(sec.content().data() + r.offset);762 const uint32_t rd = extractBits(insnPair, 32 + 11, 32 + 7);763 const uint64_t dest =764 (r.expr == R_PLT_PC ? sym.getPltVA(ctx) : sym.getVA(ctx)) + r.addend;765 const int64_t displace = dest - loc;766 767 // When the caller specifies the old value of `remove`, disallow its768 // increment.769 if (remove >= 6 && rvc && isInt<12>(displace) && rd == X_X0) {770 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;771 sec.relaxAux->writes.push_back(0xa001); // c.j772 remove = 6;773 } else if (remove >= 6 && rvc && isInt<12>(displace) && rd == X_RA &&774 !ctx.arg.is64) { // RV32C only775 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;776 sec.relaxAux->writes.push_back(0x2001); // c.jal777 remove = 6;778 } else if (remove >= 4 && isInt<21>(displace)) {779 sec.relaxAux->relocTypes[i] = R_RISCV_JAL;780 sec.relaxAux->writes.push_back(0x6f | rd << 7); // jal781 remove = 4;782 } else {783 remove = 0;784 }785}786 787// Relax local-exec TLS when hi20 is zero.788static void relaxTlsLe(Ctx &ctx, const InputSection &sec, size_t i,789 uint64_t loc, Relocation &r, uint32_t &remove) {790 uint64_t val = r.sym->getVA(ctx, r.addend);791 if (hi20(val) != 0)792 return;793 uint32_t insn = read32le(sec.content().data() + r.offset);794 switch (r.type) {795 case R_RISCV_TPREL_HI20:796 case R_RISCV_TPREL_ADD:797 // Remove lui rd, %tprel_hi(x) and add rd, rd, tp, %tprel_add(x).798 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;799 remove = 4;800 break;801 case R_RISCV_TPREL_LO12_I:802 // addi rd, rd, %tprel_lo(x) => addi rd, tp, st_value(x)803 sec.relaxAux->relocTypes[i] = R_RISCV_32;804 insn = (insn & ~(31 << 15)) | (X_TP << 15);805 sec.relaxAux->writes.push_back(setLO12_I(insn, val));806 break;807 case R_RISCV_TPREL_LO12_S:808 // sw rs, %tprel_lo(x)(rd) => sw rs, st_value(x)(rd)809 sec.relaxAux->relocTypes[i] = R_RISCV_32;810 insn = (insn & ~(31 << 15)) | (X_TP << 15);811 sec.relaxAux->writes.push_back(setLO12_S(insn, val));812 break;813 }814}815 816static void relaxHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i,817 uint64_t loc, Relocation &r, uint32_t &remove) {818 819 // Fold into use of x0+offset820 if (isInt<12>(r.sym->getVA(ctx, r.addend))) {821 switch (r.type) {822 case R_RISCV_HI20:823 // Remove lui rd, %hi20(x).824 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;825 remove = 4;826 break;827 case R_RISCV_LO12_I:828 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_I;829 break;830 case R_RISCV_LO12_S:831 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_S;832 break;833 }834 return;835 }836 837 const Defined *gp = ctx.sym.riscvGlobalPointer;838 if (!gp)839 return;840 841 if (!isInt<12>(r.sym->getVA(ctx, r.addend) - gp->getVA(ctx)))842 return;843 844 switch (r.type) {845 case R_RISCV_HI20:846 // Remove lui rd, %hi20(x).847 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;848 remove = 4;849 break;850 case R_RISCV_LO12_I:851 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_I;852 break;853 case R_RISCV_LO12_S:854 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_S;855 break;856 }857}858 859static bool relax(Ctx &ctx, int pass, InputSection &sec) {860 const uint64_t secAddr = sec.getVA();861 const MutableArrayRef<Relocation> relocs = sec.relocs();862 auto &aux = *sec.relaxAux;863 bool changed = false;864 ArrayRef<SymbolAnchor> sa = ArrayRef(aux.anchors);865 uint64_t delta = 0;866 bool tlsdescRelax = false, toLeShortForm = false;867 868 std::fill_n(aux.relocTypes.get(), relocs.size(), R_RISCV_NONE);869 aux.writes.clear();870 for (auto [i, r] : llvm::enumerate(riscv_vendor_relocs(relocs))) {871 const uint64_t loc = secAddr + r.offset - delta;872 uint32_t &cur = aux.relocDeltas[i], remove = 0;873 switch (r.type) {874 case R_RISCV_ALIGN: {875 const uint64_t nextLoc = loc + r.addend;876 const uint64_t align = PowerOf2Ceil(r.addend + 2);877 // All bytes beyond the alignment boundary should be removed.878 remove = nextLoc - ((loc + align - 1) & -align);879 // If we can't satisfy this alignment, we've found a bad input.880 if (LLVM_UNLIKELY(static_cast<int32_t>(remove) < 0)) {881 Err(ctx) << getErrorLoc(ctx, (const uint8_t *)loc)882 << "insufficient padding bytes for " << r.type << ": "883 << r.addend884 << " bytes available "885 "for requested alignment of "886 << align << " bytes";887 remove = 0;888 }889 break;890 }891 case R_RISCV_CALL:892 case R_RISCV_CALL_PLT:893 // Prevent oscillation between states by disallowing the increment of894 // `remove` after a few passes. The previous `remove` value is895 // `cur-delta`.896 if (relaxable(relocs, i)) {897 remove = pass < 4 ? 6 : cur - delta;898 relaxCall(ctx, sec, i, loc, r, remove);899 }900 break;901 case R_RISCV_TPREL_HI20:902 case R_RISCV_TPREL_ADD:903 case R_RISCV_TPREL_LO12_I:904 case R_RISCV_TPREL_LO12_S:905 if (relaxable(relocs, i))906 relaxTlsLe(ctx, sec, i, loc, r, remove);907 break;908 case R_RISCV_HI20:909 case R_RISCV_LO12_I:910 case R_RISCV_LO12_S:911 if (relaxable(relocs, i))912 relaxHi20Lo12(ctx, sec, i, loc, r, remove);913 break;914 case R_RISCV_TLSDESC_HI20:915 // For TLSDESC=>LE, we can use the short form if hi20 is zero.916 tlsdescRelax = relaxable(relocs, i);917 toLeShortForm = tlsdescRelax && r.expr == R_RELAX_TLS_GD_TO_LE &&918 !hi20(r.sym->getVA(ctx, r.addend));919 [[fallthrough]];920 case R_RISCV_TLSDESC_LOAD_LO12:921 // For TLSDESC=>LE/IE, AUIPC and L[DW] are removed if relaxable.922 if (tlsdescRelax && r.expr != R_TLSDESC_PC)923 remove = 4;924 break;925 case R_RISCV_TLSDESC_ADD_LO12:926 if (toLeShortForm)927 remove = 4;928 break;929 }930 931 // For all anchors whose offsets are <= r.offset, they are preceded by932 // the previous relocation whose `relocDeltas` value equals `delta`.933 // Decrease their st_value and update their st_size.934 for (; sa.size() && sa[0].offset <= r.offset; sa = sa.slice(1)) {935 if (sa[0].end)936 sa[0].d->size = sa[0].offset - delta - sa[0].d->value;937 else938 sa[0].d->value = sa[0].offset - delta;939 }940 delta += remove;941 if (delta != cur) {942 cur = delta;943 changed = true;944 }945 }946 947 for (const SymbolAnchor &a : sa) {948 if (a.end)949 a.d->size = a.offset - delta - a.d->value;950 else951 a.d->value = a.offset - delta;952 }953 // Inform assignAddresses that the size has changed.954 if (!isUInt<32>(delta))955 Err(ctx) << "section size decrease is too large: " << delta;956 sec.bytesDropped = delta;957 return changed;958}959 960// When relaxing just R_RISCV_ALIGN, relocDeltas is usually changed only once in961// the absence of a linker script. For call and load/store R_RISCV_RELAX, code962// shrinkage may reduce displacement and make more relocations eligible for963// relaxation. Code shrinkage may increase displacement to a call/load/store964// target at a higher fixed address, invalidating an earlier relaxation. Any965// change in section sizes can have cascading effect and require another966// relaxation pass.967bool RISCV::relaxOnce(int pass) const {968 llvm::TimeTraceScope timeScope("RISC-V relaxOnce");969 if (pass == 0)970 initSymbolAnchors(ctx);971 972 SmallVector<InputSection *, 0> storage;973 bool changed = false;974 for (OutputSection *osec : ctx.outputSections) {975 if (!(osec->flags & SHF_EXECINSTR))976 continue;977 for (InputSection *sec : getInputSections(*osec, storage))978 changed |= relax(ctx, pass, *sec);979 }980 return changed;981}982 983// If the section alignment is >= 4, advance `dot` to insert NOPs and synthesize984// an ALIGN relocation. Otherwise, return false to use default handling.985template <class ELFT, class RelTy>986bool RISCV::synthesizeAlignForInput(uint64_t &dot, InputSection *sec,987 Relocs<RelTy> rels) {988 if (!baseSec) {989 // Record the first input section with RELAX relocations. We will synthesize990 // ALIGN relocations here.991 for (auto rel : rels) {992 if (rel.getType(false) == R_RISCV_RELAX) {993 baseSec = sec;994 break;995 }996 }997 } else if (sec->addralign >= 4) {998 // If the alignment is >= 4 and the section does not start with an ALIGN999 // relocation, synthesize one.1000 bool hasAlignRel = llvm::any_of(rels, [](const RelTy &rel) {1001 return rel.r_offset == 0 && rel.getType(false) == R_RISCV_ALIGN;1002 });1003 if (!hasAlignRel) {1004 synthesizedAligns.emplace_back(dot - baseSec->getVA(),1005 sec->addralign - 2);1006 dot += sec->addralign - 2;1007 return true;1008 }1009 }1010 return false;1011}1012 1013// Finalize the relocation section by appending synthesized ALIGN relocations1014// after processing all input sections.1015template <class ELFT, class RelTy>1016void RISCV::finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,1017 Relocs<RelTy> rels) {1018 auto *f = cast<ObjFile<ELFT>>(baseSec->file);1019 auto shdr = f->template getELFShdrs<ELFT>()[baseSec->relSecIdx];1020 // Create a copy of InputSection.1021 sec = make<InputSection>(*f, shdr, baseSec->name);1022 auto *baseRelSec = cast<InputSection>(f->getSections()[baseSec->relSecIdx]);1023 *sec = *baseRelSec;1024 baseSec = nullptr;1025 1026 // Allocate buffer for original and synthesized relocations in RELA format.1027 // If CREL is used, OutputSection::finalizeNonAllocCrel will convert RELA to1028 // CREL.1029 auto newSize = rels.size() + synthesizedAligns.size();1030 auto *relas = makeThreadLocalN<typename ELFT::Rela>(newSize);1031 sec->size = newSize * sizeof(typename ELFT::Rela);1032 sec->content_ = reinterpret_cast<uint8_t *>(relas);1033 sec->type = SHT_RELA;1034 // Copy original relocations to the new buffer, potentially converting CREL to1035 // RELA.1036 for (auto [i, r] : llvm::enumerate(rels)) {1037 relas[i].r_offset = r.r_offset;1038 relas[i].setSymbolAndType(r.getSymbol(0), r.getType(0), false);1039 if constexpr (RelTy::HasAddend)1040 relas[i].r_addend = r.r_addend;1041 }1042 // Append synthesized ALIGN relocations to the buffer.1043 for (auto [i, r] : llvm::enumerate(synthesizedAligns)) {1044 auto &rela = relas[rels.size() + i];1045 rela.r_offset = r.first;1046 rela.setSymbolAndType(0, R_RISCV_ALIGN, false);1047 rela.r_addend = r.second;1048 }1049 synthesizedAligns.clear();1050 // Replace the old relocation section with the new one in the output section.1051 // addOrphanSections ensures that the output relocation section is processed1052 // after osec.1053 for (SectionCommand *cmd : sec->getParent()->commands) {1054 auto *isd = dyn_cast<InputSectionDescription>(cmd);1055 if (!isd)1056 continue;1057 for (auto *&isec : isd->sections)1058 if (isec == baseRelSec)1059 isec = sec;1060 }1061}1062 1063template <class ELFT>1064bool RISCV::synthesizeAlignAux(uint64_t &dot, InputSection *sec) {1065 bool ret = false;1066 if (sec) {1067 invokeOnRelocs(*sec, ret = synthesizeAlignForInput<ELFT>, dot, sec);1068 } else if (baseSec) {1069 invokeOnRelocs(*baseSec, finalizeSynthesizeAligns<ELFT>, dot, sec);1070 }1071 return ret;1072}1073 1074// Without linker relaxation enabled for a particular relocatable file or1075// section, the assembler will not generate R_RISCV_ALIGN relocations for1076// alignment directives. This becomes problematic in a two-stage linking1077// process: ld -r a.o b.o -o ab.o; ld ab.o -o ab. This function synthesizes an1078// R_RISCV_ALIGN relocation at section start when needed.1079//1080// When called with an input section (`sec` is not null): If the section1081// alignment is >= 4, advance `dot` to insert NOPs and synthesize an ALIGN1082// relocation.1083//1084// When called after all input sections are processed (`sec` is null): The1085// output relocation section is updated with all the newly synthesized ALIGN1086// relocations.1087bool RISCV::synthesizeAlign(uint64_t &dot, InputSection *sec) {1088 assert(ctx.arg.relocatable);1089 if (ctx.arg.is64)1090 return synthesizeAlignAux<ELF64LE>(dot, sec);1091 return synthesizeAlignAux<ELF32LE>(dot, sec);1092}1093 1094void RISCV::finalizeRelax(int passes) const {1095 llvm::TimeTraceScope timeScope("Finalize RISC-V relaxation");1096 Log(ctx) << "relaxation passes: " << passes;1097 SmallVector<InputSection *, 0> storage;1098 for (OutputSection *osec : ctx.outputSections) {1099 if (!(osec->flags & SHF_EXECINSTR))1100 continue;1101 for (InputSection *sec : getInputSections(*osec, storage)) {1102 RelaxAux &aux = *sec->relaxAux;1103 if (!aux.relocDeltas)1104 continue;1105 1106 MutableArrayRef<Relocation> rels = sec->relocs();1107 ArrayRef<uint8_t> old = sec->content();1108 size_t newSize = old.size() - aux.relocDeltas[rels.size() - 1];1109 size_t writesIdx = 0;1110 uint8_t *p = ctx.bAlloc.Allocate<uint8_t>(newSize);1111 uint64_t offset = 0;1112 int64_t delta = 0;1113 sec->content_ = p;1114 sec->size = newSize;1115 sec->bytesDropped = 0;1116 1117 // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite1118 // instructions for relaxed relocations.1119 for (size_t i = 0, e = rels.size(); i != e; ++i) {1120 uint32_t remove = aux.relocDeltas[i] - delta;1121 delta = aux.relocDeltas[i];1122 if (remove == 0 && aux.relocTypes[i] == R_RISCV_NONE)1123 continue;1124 1125 // Copy from last location to the current relocated location.1126 const Relocation &r = rels[i];1127 uint64_t size = r.offset - offset;1128 memcpy(p, old.data() + offset, size);1129 p += size;1130 1131 // For R_RISCV_ALIGN, we will place `offset` in a location (among NOPs)1132 // to satisfy the alignment requirement. If both `remove` and r.addend1133 // are multiples of 4, it is as if we have skipped some NOPs. Otherwise1134 // we are in the middle of a 4-byte NOP, and we need to rewrite the NOP1135 // sequence.1136 int64_t skip = 0;1137 if (r.type == R_RISCV_ALIGN) {1138 if (remove % 4 || r.addend % 4) {1139 skip = r.addend - remove;1140 int64_t j = 0;1141 for (; j + 4 <= skip; j += 4)1142 write32le(p + j, 0x00000013); // nop1143 if (j != skip) {1144 assert(j + 2 == skip);1145 write16le(p + j, 0x0001); // c.nop1146 }1147 }1148 } else if (RelType newType = aux.relocTypes[i]) {1149 switch (newType) {1150 case INTERNAL_R_RISCV_GPREL_I:1151 case INTERNAL_R_RISCV_GPREL_S:1152 case INTERNAL_R_RISCV_X0REL_I:1153 case INTERNAL_R_RISCV_X0REL_S:1154 break;1155 case R_RISCV_RELAX:1156 // Used by relaxTlsLe to indicate the relocation is ignored.1157 break;1158 case R_RISCV_RVC_JUMP:1159 skip = 2;1160 write16le(p, aux.writes[writesIdx++]);1161 break;1162 case R_RISCV_JAL:1163 skip = 4;1164 write32le(p, aux.writes[writesIdx++]);1165 break;1166 case R_RISCV_32:1167 // Used by relaxTlsLe to write a uint32_t then suppress the handling1168 // in relocateAlloc.1169 skip = 4;1170 write32le(p, aux.writes[writesIdx++]);1171 aux.relocTypes[i] = R_RISCV_NONE;1172 break;1173 default:1174 llvm_unreachable("unsupported type");1175 }1176 }1177 1178 p += skip;1179 offset = r.offset + skip + remove;1180 }1181 memcpy(p, old.data() + offset, old.size() - offset);1182 1183 // Subtract the previous relocDeltas value from the relocation offset.1184 // For a pair of R_RISCV_CALL/R_RISCV_RELAX with the same offset, decrease1185 // their r_offset by the same delta.1186 delta = 0;1187 for (size_t i = 0, e = rels.size(); i != e;) {1188 uint64_t cur = rels[i].offset;1189 do {1190 rels[i].offset -= delta;1191 if (aux.relocTypes[i] != R_RISCV_NONE)1192 rels[i].type = aux.relocTypes[i];1193 } while (++i != e && rels[i].offset == cur);1194 delta = aux.relocDeltas[i - 1];1195 }1196 }1197 }1198}1199 1200namespace {1201// Representation of the merged .riscv.attributes input sections. The psABI1202// specifies merge policy for attributes. E.g. if we link an object without an1203// extension with an object with the extension, the output Tag_RISCV_arch shall1204// contain the extension. Some tools like objdump parse .riscv.attributes and1205// disabling some instructions if the first Tag_RISCV_arch does not contain an1206// extension.1207class RISCVAttributesSection final : public SyntheticSection {1208public:1209 RISCVAttributesSection(Ctx &ctx)1210 : SyntheticSection(ctx, ".riscv.attributes", SHT_RISCV_ATTRIBUTES, 0, 1) {1211 }1212 1213 size_t getSize() const override { return size; }1214 void writeTo(uint8_t *buf) override;1215 1216 static constexpr StringRef vendor = "riscv";1217 DenseMap<unsigned, unsigned> intAttr;1218 DenseMap<unsigned, StringRef> strAttr;1219 size_t size = 0;1220};1221} // namespace1222 1223static void mergeArch(Ctx &ctx, RISCVISAUtils::OrderedExtensionMap &mergedExts,1224 unsigned &mergedXlen, const InputSectionBase *sec,1225 StringRef s) {1226 auto maybeInfo = RISCVISAInfo::parseNormalizedArchString(s);1227 if (!maybeInfo) {1228 Err(ctx) << sec << ": " << s << ": " << maybeInfo.takeError();1229 return;1230 }1231 1232 // Merge extensions.1233 RISCVISAInfo &info = **maybeInfo;1234 if (mergedExts.empty()) {1235 mergedExts = info.getExtensions();1236 mergedXlen = info.getXLen();1237 } else {1238 for (const auto &ext : info.getExtensions()) {1239 auto p = mergedExts.insert(ext);1240 if (!p.second) {1241 if (std::tie(p.first->second.Major, p.first->second.Minor) <1242 std::tie(ext.second.Major, ext.second.Minor))1243 p.first->second = ext.second;1244 }1245 }1246 }1247}1248 1249static void mergeAtomic(Ctx &ctx, DenseMap<unsigned, unsigned>::iterator it,1250 const InputSectionBase *oldSection,1251 const InputSectionBase *newSection,1252 RISCVAttrs::RISCVAtomicAbiTag oldTag,1253 RISCVAttrs::RISCVAtomicAbiTag newTag) {1254 using RISCVAttrs::RISCVAtomicAbiTag;1255 // Same tags stay the same, and UNKNOWN is compatible with anything1256 if (oldTag == newTag || newTag == RISCVAtomicAbiTag::UNKNOWN)1257 return;1258 1259 auto reportAbiError = [&]() {1260 Err(ctx) << "atomic abi mismatch for " << oldSection->name << "\n>>> "1261 << oldSection << ": atomic_abi=" << static_cast<unsigned>(oldTag)1262 << "\n>>> " << newSection1263 << ": atomic_abi=" << static_cast<unsigned>(newTag);1264 };1265 1266 auto reportUnknownAbiError = [&](const InputSectionBase *section,1267 RISCVAtomicAbiTag tag) {1268 switch (tag) {1269 case RISCVAtomicAbiTag::UNKNOWN:1270 case RISCVAtomicAbiTag::A6C:1271 case RISCVAtomicAbiTag::A6S:1272 case RISCVAtomicAbiTag::A7:1273 return;1274 };1275 Err(ctx) << "unknown atomic abi for " << section->name << "\n>>> "1276 << section << ": atomic_abi=" << static_cast<unsigned>(tag);1277 };1278 switch (oldTag) {1279 case RISCVAtomicAbiTag::UNKNOWN:1280 it->getSecond() = static_cast<unsigned>(newTag);1281 return;1282 case RISCVAtomicAbiTag::A6C:1283 switch (newTag) {1284 case RISCVAtomicAbiTag::A6S:1285 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C);1286 return;1287 case RISCVAtomicAbiTag::A7:1288 reportAbiError();1289 return;1290 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:1291 case RISCVAttrs::RISCVAtomicAbiTag::A6C:1292 return;1293 };1294 break;1295 1296 case RISCVAtomicAbiTag::A6S:1297 switch (newTag) {1298 case RISCVAtomicAbiTag::A6C:1299 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C);1300 return;1301 case RISCVAtomicAbiTag::A7:1302 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7);1303 return;1304 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:1305 case RISCVAttrs::RISCVAtomicAbiTag::A6S:1306 return;1307 };1308 break;1309 1310 case RISCVAtomicAbiTag::A7:1311 switch (newTag) {1312 case RISCVAtomicAbiTag::A6S:1313 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7);1314 return;1315 case RISCVAtomicAbiTag::A6C:1316 reportAbiError();1317 return;1318 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:1319 case RISCVAttrs::RISCVAtomicAbiTag::A7:1320 return;1321 };1322 break;1323 };1324 1325 // If we get here, then we have an invalid tag, so report it.1326 // Putting these checks at the end allows us to only do these checks when we1327 // need to, since this is expected to be a rare occurrence.1328 reportUnknownAbiError(oldSection, oldTag);1329 reportUnknownAbiError(newSection, newTag);1330}1331 1332static RISCVAttributesSection *1333mergeAttributesSection(Ctx &ctx,1334 const SmallVector<InputSectionBase *, 0> §ions) {1335 using RISCVAttrs::RISCVAtomicAbiTag;1336 RISCVISAUtils::OrderedExtensionMap exts;1337 const InputSectionBase *firstStackAlign = nullptr;1338 const InputSectionBase *firstAtomicAbi = nullptr;1339 unsigned firstStackAlignValue = 0, xlen = 0;1340 bool hasArch = false;1341 1342 ctx.in.riscvAttributes = std::make_unique<RISCVAttributesSection>(ctx);1343 auto &merged = static_cast<RISCVAttributesSection &>(*ctx.in.riscvAttributes);1344 1345 // Collect all tags values from attributes section.1346 const auto &attributesTags = RISCVAttrs::getRISCVAttributeTags();1347 for (const InputSectionBase *sec : sections) {1348 RISCVAttributeParser parser;1349 if (Error e = parser.parse(sec->content(), llvm::endianness::little))1350 Warn(ctx) << sec << ": " << std::move(e);1351 for (const auto &tag : attributesTags) {1352 switch (RISCVAttrs::AttrType(tag.attr)) {1353 // Integer attributes.1354 case RISCVAttrs::STACK_ALIGN:1355 if (auto i = parser.getAttributeValue(tag.attr)) {1356 auto r = merged.intAttr.try_emplace(tag.attr, *i);1357 if (r.second) {1358 firstStackAlign = sec;1359 firstStackAlignValue = *i;1360 } else if (r.first->second != *i) {1361 Err(ctx) << sec << " has stack_align=" << *i << " but "1362 << firstStackAlign1363 << " has stack_align=" << firstStackAlignValue;1364 }1365 }1366 continue;1367 case RISCVAttrs::UNALIGNED_ACCESS:1368 if (auto i = parser.getAttributeValue(tag.attr))1369 merged.intAttr[tag.attr] |= *i;1370 continue;1371 1372 // String attributes.1373 case RISCVAttrs::ARCH:1374 if (auto s = parser.getAttributeString(tag.attr)) {1375 hasArch = true;1376 mergeArch(ctx, exts, xlen, sec, *s);1377 }1378 continue;1379 1380 // Attributes which use the default handling.1381 case RISCVAttrs::PRIV_SPEC:1382 case RISCVAttrs::PRIV_SPEC_MINOR:1383 case RISCVAttrs::PRIV_SPEC_REVISION:1384 break;1385 1386 case RISCVAttrs::AttrType::ATOMIC_ABI:1387 if (auto i = parser.getAttributeValue(tag.attr)) {1388 auto r = merged.intAttr.try_emplace(tag.attr, *i);1389 if (r.second)1390 firstAtomicAbi = sec;1391 else1392 mergeAtomic(ctx, r.first, firstAtomicAbi, sec,1393 static_cast<RISCVAtomicAbiTag>(r.first->getSecond()),1394 static_cast<RISCVAtomicAbiTag>(*i));1395 }1396 continue;1397 }1398 1399 // Fallback for deprecated priv_spec* and other unknown attributes: retain1400 // the attribute if all input sections agree on the value. GNU ld uses 01401 // and empty strings as default values which are not dumped to the output.1402 // TODO Adjust after resolution to1403 // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/issues/3521404 if (tag.attr % 2 == 0) {1405 if (auto i = parser.getAttributeValue(tag.attr)) {1406 auto r = merged.intAttr.try_emplace(tag.attr, *i);1407 if (!r.second && r.first->second != *i)1408 r.first->second = 0;1409 }1410 } else if (auto s = parser.getAttributeString(tag.attr)) {1411 auto r = merged.strAttr.try_emplace(tag.attr, *s);1412 if (!r.second && r.first->second != *s)1413 r.first->second = {};1414 }1415 }1416 }1417 1418 if (hasArch && xlen != 0) {1419 if (auto result = RISCVISAInfo::createFromExtMap(xlen, exts)) {1420 merged.strAttr.try_emplace(RISCVAttrs::ARCH,1421 ctx.saver.save((*result)->toString()));1422 } else {1423 Err(ctx) << result.takeError();1424 }1425 }1426 1427 // The total size of headers: format-version [ <section-length> "vendor-name"1428 // [ <file-tag> <size>.1429 size_t size = 5 + merged.vendor.size() + 1 + 5;1430 for (auto &attr : merged.intAttr)1431 if (attr.second != 0)1432 size += getULEB128Size(attr.first) + getULEB128Size(attr.second);1433 for (auto &attr : merged.strAttr)1434 if (!attr.second.empty())1435 size += getULEB128Size(attr.first) + attr.second.size() + 1;1436 merged.size = size;1437 return &merged;1438}1439 1440void RISCVAttributesSection::writeTo(uint8_t *buf) {1441 const size_t size = getSize();1442 uint8_t *const end = buf + size;1443 *buf = ELFAttrs::Format_Version;1444 write32(ctx, buf + 1, size - 1);1445 buf += 5;1446 1447 memcpy(buf, vendor.data(), vendor.size());1448 buf += vendor.size() + 1;1449 1450 *buf = ELFAttrs::File;1451 write32(ctx, buf + 1, end - buf);1452 buf += 5;1453 1454 for (auto &attr : intAttr) {1455 if (attr.second == 0)1456 continue;1457 buf += encodeULEB128(attr.first, buf);1458 buf += encodeULEB128(attr.second, buf);1459 }1460 for (auto &attr : strAttr) {1461 if (attr.second.empty())1462 continue;1463 buf += encodeULEB128(attr.first, buf);1464 memcpy(buf, attr.second.data(), attr.second.size());1465 buf += attr.second.size() + 1;1466 }1467}1468 1469void elf::mergeRISCVAttributesSections(Ctx &ctx) {1470 // Find the first input SHT_RISCV_ATTRIBUTES; return if not found.1471 size_t place =1472 llvm::find_if(ctx.inputSections,1473 [](auto *s) { return s->type == SHT_RISCV_ATTRIBUTES; }) -1474 ctx.inputSections.begin();1475 if (place == ctx.inputSections.size())1476 return;1477 1478 // Extract all SHT_RISCV_ATTRIBUTES sections into `sections`.1479 SmallVector<InputSectionBase *, 0> sections;1480 llvm::erase_if(ctx.inputSections, [&](InputSectionBase *s) {1481 if (s->type != SHT_RISCV_ATTRIBUTES)1482 return false;1483 sections.push_back(s);1484 return true;1485 });1486 1487 // Add the merged section.1488 ctx.inputSections.insert(ctx.inputSections.begin() + place,1489 mergeAttributesSection(ctx, sections));1490}1491 1492void elf::setRISCVTargetInfo(Ctx &ctx) { ctx.target.reset(new RISCV(ctx)); }1493 1494template <class ELFT, class RelTy>1495void RISCV::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {1496 RelocScan rs(ctx, &sec);1497 // Many relocations end up in sec.relocations.1498 sec.relocations.reserve(rels.size());1499 1500 StringRef rvVendor;1501 for (auto it = rels.begin(); it != rels.end(); ++it) {1502 RelType type = it->getType(false);1503 uint32_t symIndex = it->getSymbol(false);1504 Symbol &sym = sec.getFile<ELFT>()->getSymbol(symIndex);1505 const uint8_t *loc = sec.content().data() + it->r_offset;1506 1507 if (type == R_RISCV_VENDOR) {1508 if (!rvVendor.empty())1509 Err(ctx) << getErrorLoc(ctx, loc)1510 << "malformed consecutive R_RISCV_VENDOR relocations";1511 rvVendor = sym.getName();1512 continue;1513 } else if (!rvVendor.empty()) {1514 uint32_t VendorFlag = getRISCVVendorRelMarker(rvVendor);1515 if (!VendorFlag) {1516 Err(ctx) << getErrorLoc(ctx, loc)1517 << "unknown vendor-specific relocation (" << type.v1518 << ") in namespace '" << rvVendor << "' against symbol '"1519 << &sym << "'";1520 rvVendor = "";1521 continue;1522 }1523 1524 rvVendor = "";1525 assert((type.v < 256) && "Out of range relocation detected!");1526 type.v |= VendorFlag;1527 }1528 1529 rs.scan<ELFT, RelTy>(it, type, rs.getAddend<ELFT>(*it, type));1530 }1531 1532 // Sort relocations by offset for more efficient searching for1533 // R_RISCV_PCREL_HI20.1534 llvm::stable_sort(sec.relocs(),1535 [](const Relocation &lhs, const Relocation &rhs) {1536 return lhs.offset < rhs.offset;1537 });1538}1539 1540template <class ELFT> void RISCV::scanSection1(InputSectionBase &sec) {1541 const RelsOrRelas<ELFT> rels = sec.template relsOrRelas<ELFT>();1542 if (rels.areRelocsCrel())1543 scanSectionImpl<ELFT>(sec, rels.crels);1544 else1545 scanSectionImpl<ELFT>(sec, rels.relas);1546}1547 1548void RISCV::scanSection(InputSectionBase &sec) {1549 invokeELFT(scanSection1, sec);1550}1551 1552namespace lld::elf {1553uint32_t getRISCVVendorRelMarker(StringRef rvVendor) {1554 return StringSwitch<uint32_t>(rvVendor)1555 .Case("QUALCOMM", INTERNAL_RISCV_VENDOR_QUALCOMM)1556 .Case("ANDES", INTERNAL_RISCV_VENDOR_ANDES)1557 .Default(0);1558}1559 1560std::optional<StringRef> getRISCVVendorString(RelType ty) {1561 if ((ty.v & INTERNAL_RISCV_VENDOR_MASK) == INTERNAL_RISCV_VENDOR_QUALCOMM)1562 return "QUALCOMM";1563 if ((ty.v & INTERNAL_RISCV_VENDOR_MASK) == INTERNAL_RISCV_VENDOR_ANDES)1564 return "ANDES";1565 return std::nullopt;1566}1567 1568} // namespace lld::elf1569