508 lines · cpp
1//===-- InstrProfCorrelator.cpp -------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ProfileData/InstrProfCorrelator.h"10#include "llvm/DebugInfo/DIContext.h"11#include "llvm/DebugInfo/DWARF/DWARFContext.h"12#include "llvm/DebugInfo/DWARF/DWARFDie.h"13#include "llvm/DebugInfo/DWARF/DWARFFormValue.h"14#include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"15#include "llvm/DebugInfo/DWARF/DWARFUnit.h"16#include "llvm/DebugInfo/DWARF/LowLevel/DWARFExpression.h"17#include "llvm/Object/MachO.h"18#include "llvm/Support/Debug.h"19#include "llvm/Support/Format.h"20#include "llvm/Support/WithColor.h"21#include <optional>22 23#define DEBUG_TYPE "correlator"24 25using namespace llvm;26 27/// Get profile section.28static Expected<object::SectionRef>29getInstrProfSection(const object::ObjectFile &Obj, InstrProfSectKind IPSK) {30 // On COFF, the getInstrProfSectionName returns the section names may followed31 // by "$M". The linker removes the dollar and everything after it in the final32 // binary. Do the same to match.33 Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat();34 auto StripSuffix = [ObjFormat](StringRef N) {35 return ObjFormat == Triple::COFF ? N.split('$').first : N;36 };37 std::string ExpectedSectionName =38 getInstrProfSectionName(IPSK, ObjFormat,39 /*AddSegmentInfo=*/false);40 ExpectedSectionName = StripSuffix(ExpectedSectionName);41 for (auto &Section : Obj.sections()) {42 if (auto SectionName = Section.getName())43 if (*SectionName == ExpectedSectionName)44 return Section;45 }46 return make_error<InstrProfError>(47 instrprof_error::unable_to_correlate_profile,48 "could not find section (" + Twine(ExpectedSectionName) + ")");49}50 51const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";52const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";53const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";54 55llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>56InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,57 const object::ObjectFile &Obj,58 ProfCorrelatorKind FileKind) {59 auto C = std::make_unique<Context>();60 auto CountersSection = getInstrProfSection(Obj, IPSK_cnts);61 if (auto Err = CountersSection.takeError())62 return std::move(Err);63 if (FileKind == InstrProfCorrelator::BINARY) {64 auto DataSection = getInstrProfSection(Obj, IPSK_covdata);65 if (auto Err = DataSection.takeError())66 return std::move(Err);67 auto DataOrErr = DataSection->getContents();68 if (!DataOrErr)69 return DataOrErr.takeError();70 auto NameSection = getInstrProfSection(Obj, IPSK_covname);71 if (auto Err = NameSection.takeError())72 return std::move(Err);73 auto NameOrErr = NameSection->getContents();74 if (!NameOrErr)75 return NameOrErr.takeError();76 C->DataStart = DataOrErr->data();77 C->DataEnd = DataOrErr->data() + DataOrErr->size();78 C->NameStart = NameOrErr->data();79 C->NameSize = NameOrErr->size();80 }81 C->Buffer = std::move(Buffer);82 C->CountersSectionStart = CountersSection->getAddress();83 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();84 // In COFF object file, there's a null byte at the beginning of the counter85 // section which doesn't exist in raw profile.86 if (Obj.getTripleObjectFormat() == Triple::COFF)87 ++C->CountersSectionStart;88 89 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;90 return Expected<std::unique_ptr<Context>>(std::move(C));91}92 93llvm::Expected<std::unique_ptr<InstrProfCorrelator>>94InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind,95 const object::BuildIDFetcher *BIDFetcher,96 const ArrayRef<object::BuildID> BIs) {97 std::optional<std::string> Path;98 if (BIDFetcher) {99 if (BIs.empty())100 return make_error<InstrProfError>(101 instrprof_error::unable_to_correlate_profile,102 "unsupported profile binary correlation when there is no build ID "103 "in a profile");104 if (BIs.size() > 1)105 return make_error<InstrProfError>(106 instrprof_error::unable_to_correlate_profile,107 "unsupported profile binary correlation when there are multiple "108 "build IDs in a profile");109 110 Path = BIDFetcher->fetch(BIs.front());111 if (!Path)112 return make_error<InstrProfError>(113 instrprof_error::unable_to_correlate_profile,114 "Missing build ID: " + llvm::toHex(BIs.front(),115 /*LowerCase=*/true));116 Filename = *Path;117 }118 119 if (FileKind == DEBUG_INFO) {120 auto DsymObjectsOrErr =121 object::MachOObjectFile::findDsymObjectMembers(Filename);122 if (auto Err = DsymObjectsOrErr.takeError())123 return std::move(Err);124 if (!DsymObjectsOrErr->empty()) {125 // TODO: Enable profile correlation when there are multiple objects in a126 // dSYM bundle.127 if (DsymObjectsOrErr->size() > 1)128 return make_error<InstrProfError>(129 instrprof_error::unable_to_correlate_profile,130 "using multiple objects is not yet supported");131 Filename = *DsymObjectsOrErr->begin();132 }133 auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename));134 if (auto Err = BufferOrErr.takeError())135 return std::move(Err);136 137 return get(std::move(*BufferOrErr), FileKind);138 }139 if (FileKind == BINARY) {140 auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename));141 if (auto Err = BufferOrErr.takeError())142 return std::move(Err);143 144 return get(std::move(*BufferOrErr), FileKind);145 }146 return make_error<InstrProfError>(147 instrprof_error::unable_to_correlate_profile,148 "unsupported correlation kind (only DWARF debug info and Binary format "149 "(ELF/COFF) are supported)");150}151 152llvm::Expected<std::unique_ptr<InstrProfCorrelator>>153InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer,154 ProfCorrelatorKind FileKind) {155 auto BinOrErr = object::createBinary(*Buffer);156 if (auto Err = BinOrErr.takeError())157 return std::move(Err);158 159 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {160 auto CtxOrErr = Context::get(std::move(Buffer), *Obj, FileKind);161 if (auto Err = CtxOrErr.takeError())162 return std::move(Err);163 auto T = Obj->makeTriple();164 if (T.isArch64Bit())165 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj,166 FileKind);167 if (T.isArch32Bit())168 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj,169 FileKind);170 }171 return make_error<InstrProfError>(172 instrprof_error::unable_to_correlate_profile, "not an object file");173}174 175std::optional<size_t> InstrProfCorrelator::getDataSize() const {176 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this))177 return C->getDataSize();178 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this))179 return C->getDataSize();180 return {};181}182 183namespace llvm {184 185template <>186InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(187 std::unique_ptr<InstrProfCorrelator::Context> Ctx)188 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,189 std::move(Ctx)) {}190template <>191InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(192 std::unique_ptr<InstrProfCorrelator::Context> Ctx)193 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,194 std::move(Ctx)) {}195template <>196bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {197 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;198}199template <>200bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {201 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;202}203 204} // end namespace llvm205 206template <class IntPtrT>207llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>208InstrProfCorrelatorImpl<IntPtrT>::get(209 std::unique_ptr<InstrProfCorrelator::Context> Ctx,210 const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) {211 if (FileKind == DEBUG_INFO) {212 if (Obj.isELF() || Obj.isMachO()) {213 auto DICtx = DWARFContext::create(Obj);214 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(215 std::move(DICtx), std::move(Ctx));216 }217 return make_error<InstrProfError>(218 instrprof_error::unable_to_correlate_profile,219 "unsupported debug info format (only DWARF is supported)");220 }221 if (Obj.isELF() || Obj.isCOFF())222 return std::make_unique<BinaryInstrProfCorrelator<IntPtrT>>(std::move(Ctx));223 return make_error<InstrProfError>(224 instrprof_error::unable_to_correlate_profile,225 "unsupported binary format (only ELF and COFF are supported)");226}227 228template <class IntPtrT>229Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {230 assert(Data.empty() && Names.empty() && NamesVec.empty());231 correlateProfileDataImpl(MaxWarnings);232 if (this->Data.empty())233 return make_error<InstrProfError>(234 instrprof_error::unable_to_correlate_profile,235 "could not find any profile data metadata in correlated file");236 Error Result = correlateProfileNameImpl();237 this->CounterOffsets.clear();238 this->NamesVec.clear();239 return Result;240}241 242template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {243 static void mapping(yaml::IO &io,244 InstrProfCorrelator::CorrelationData &Data) {245 io.mapRequired("Probes", Data.Probes);246 }247};248 249template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {250 static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {251 io.mapRequired("Function Name", P.FunctionName);252 io.mapOptional("Linkage Name", P.LinkageName);253 io.mapRequired("CFG Hash", P.CFGHash);254 io.mapRequired("Counter Offset", P.CounterOffset);255 io.mapRequired("Num Counters", P.NumCounters);256 io.mapOptional("File", P.FilePath);257 io.mapOptional("Line", P.LineNumber);258 }259};260 261template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {262 static const bool flow = false;263};264 265template <class IntPtrT>266Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,267 raw_ostream &OS) {268 InstrProfCorrelator::CorrelationData Data;269 correlateProfileDataImpl(MaxWarnings, &Data);270 if (Data.Probes.empty())271 return make_error<InstrProfError>(272 instrprof_error::unable_to_correlate_profile,273 "could not find any profile data metadata in debug info");274 yaml::Output YamlOS(OS);275 YamlOS << Data;276 return Error::success();277}278 279template <class IntPtrT>280void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,281 uint64_t CFGHash,282 IntPtrT CounterOffset,283 IntPtrT FunctionPtr,284 uint32_t NumCounters) {285 // Check if a probe was already added for this counter offset.286 if (!CounterOffsets.insert(CounterOffset).second)287 return;288 Data.push_back({289 maybeSwap<uint64_t>(NameRef),290 maybeSwap<uint64_t>(CFGHash),291 // In this mode, CounterPtr actually stores the section relative address292 // of the counter.293 maybeSwap<IntPtrT>(CounterOffset),294 // TODO: MC/DC is not yet supported.295 /*BitmapOffset=*/maybeSwap<IntPtrT>(0),296 maybeSwap<IntPtrT>(FunctionPtr),297 // TODO: Value profiling is not yet supported.298 /*ValuesPtr=*/maybeSwap<IntPtrT>(0),299 maybeSwap<uint32_t>(NumCounters),300 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},301 // TODO: MC/DC is not yet supported.302 /*NumBitmapBytes=*/maybeSwap<uint32_t>(0),303 });304}305 306template <class IntPtrT>307std::optional<uint64_t>308DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {309 auto Locations = Die.getLocations(dwarf::DW_AT_location);310 if (!Locations) {311 consumeError(Locations.takeError());312 return {};313 }314 auto &DU = *Die.getDwarfUnit();315 auto AddressSize = DU.getAddressByteSize();316 for (auto &Location : *Locations) {317 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);318 DWARFExpression Expr(Data, AddressSize);319 for (auto &Op : Expr) {320 if (Op.getCode() == dwarf::DW_OP_addr)321 return Op.getRawOperand(0);322 if (Op.getCode() == dwarf::DW_OP_addrx) {323 uint64_t Index = Op.getRawOperand(0);324 if (auto SA = DU.getAddrOffsetSectionItem(Index))325 return SA->Address;326 }327 }328 }329 return {};330}331 332template <class IntPtrT>333bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {334 const auto &ParentDie = Die.getParent();335 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())336 return false;337 if (Die.getTag() != dwarf::DW_TAG_variable)338 return false;339 if (!ParentDie.isSubprogramDIE())340 return false;341 if (!Die.hasChildren())342 return false;343 if (const char *Name = Die.getName(DINameKind::ShortName))344 return StringRef(Name).starts_with(getInstrProfCountersVarPrefix());345 return false;346}347 348template <class IntPtrT>349void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(350 int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {351 bool UnlimitedWarnings = (MaxWarnings == 0);352 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings353 int NumSuppressedWarnings = -MaxWarnings;354 auto MaybeAddProbe = [&](DWARFDie Die) {355 if (!isDIEOfProbe(Die))356 return;357 std::optional<const char *> FunctionName;358 std::optional<uint64_t> CFGHash;359 std::optional<uint64_t> CounterPtr = getLocation(Die);360 auto FnDie = Die.getParent();361 auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));362 std::optional<uint64_t> NumCounters;363 for (const DWARFDie &Child : Die.children()) {364 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)365 continue;366 auto AnnotationFormName = Child.find(dwarf::DW_AT_name);367 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);368 if (!AnnotationFormName || !AnnotationFormValue)369 continue;370 auto AnnotationNameOrErr = AnnotationFormName->getAsCString();371 if (auto Err = AnnotationNameOrErr.takeError()) {372 consumeError(std::move(Err));373 continue;374 }375 StringRef AnnotationName = *AnnotationNameOrErr;376 if (AnnotationName == InstrProfCorrelator::FunctionNameAttributeName) {377 if (auto EC =378 AnnotationFormValue->getAsCString().moveInto(FunctionName))379 consumeError(std::move(EC));380 } else if (AnnotationName == InstrProfCorrelator::CFGHashAttributeName) {381 CFGHash = AnnotationFormValue->getAsUnsignedConstant();382 } else if (AnnotationName ==383 InstrProfCorrelator::NumCountersAttributeName) {384 NumCounters = AnnotationFormValue->getAsUnsignedConstant();385 }386 }387 // If there is no function and no counter, assume it was dead-stripped388 if (!FunctionPtr && !CounterPtr)389 return;390 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {391 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {392 WithColor::warning()393 << "Incomplete DIE for function " << FunctionName394 << ": CFGHash=" << CFGHash << " CounterPtr=" << CounterPtr395 << " NumCounters=" << NumCounters << "\n";396 LLVM_DEBUG(Die.dump(dbgs()));397 }398 return;399 }400 uint64_t CountersStart = this->Ctx->CountersSectionStart;401 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;402 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {403 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {404 WithColor::warning()405 << format("CounterPtr out of range for function %s: Actual=0x%x "406 "Expected=[0x%x, 0x%x)\n",407 *FunctionName, *CounterPtr, CountersStart, CountersEnd);408 LLVM_DEBUG(Die.dump(dbgs()));409 }410 return;411 }412 if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {413 WithColor::warning() << format("Could not find address of function %s\n",414 *FunctionName);415 LLVM_DEBUG(Die.dump(dbgs()));416 }417 // In debug info correlation mode, the CounterPtr is an absolute address of418 // the counter, but it's expected to be relative later when iterating Data.419 IntPtrT CounterOffset = *CounterPtr - CountersStart;420 if (Data) {421 InstrProfCorrelator::Probe P;422 P.FunctionName = *FunctionName;423 if (const char *Name = FnDie.getName(DINameKind::LinkageName))424 P.LinkageName = Name;425 P.CFGHash = *CFGHash;426 P.CounterOffset = CounterOffset;427 P.NumCounters = *NumCounters;428 auto FilePath = FnDie.getDeclFile(429 DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);430 if (!FilePath.empty())431 P.FilePath = FilePath;432 if (auto LineNumber = FnDie.getDeclLine())433 P.LineNumber = LineNumber;434 Data->Probes.push_back(P);435 } else {436 this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash,437 CounterOffset, FunctionPtr.value_or(0), *NumCounters);438 this->NamesVec.push_back(*FunctionName);439 }440 };441 for (auto &CU : DICtx->normal_units())442 for (const auto &Entry : CU->dies())443 MaybeAddProbe(DWARFDie(CU.get(), &Entry));444 for (auto &CU : DICtx->dwo_units())445 for (const auto &Entry : CU->dies())446 MaybeAddProbe(DWARFDie(CU.get(), &Entry));447 448 if (!UnlimitedWarnings && NumSuppressedWarnings > 0)449 WithColor::warning() << format("Suppressed %d additional warnings\n",450 NumSuppressedWarnings);451}452 453template <class IntPtrT>454Error DwarfInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {455 if (this->NamesVec.empty()) {456 return make_error<InstrProfError>(457 instrprof_error::unable_to_correlate_profile,458 "could not find any profile name metadata in debug info");459 }460 auto Result =461 collectGlobalObjectNameStrings(this->NamesVec,462 /*doCompression=*/false, this->Names);463 return Result;464}465 466template <class IntPtrT>467void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(468 int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) {469 using RawProfData = RawInstrProf::ProfileData<IntPtrT>;470 bool UnlimitedWarnings = (MaxWarnings == 0);471 // -N suppressed warnings means we can emit up to N (unsuppressed) warnings472 int NumSuppressedWarnings = -MaxWarnings;473 474 const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart;475 const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd;476 // We need to use < here because the last data record may have no padding.477 for (const RawProfData *I = DataStart; I < DataEnd; ++I) {478 uint64_t CounterPtr = this->template maybeSwap<IntPtrT>(I->CounterPtr);479 uint64_t CountersStart = this->Ctx->CountersSectionStart;480 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;481 if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) {482 if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {483 WithColor::warning()484 << format("CounterPtr out of range for function: Actual=0x%x "485 "Expected=[0x%x, 0x%x) at data offset=0x%x\n",486 CounterPtr, CountersStart, CountersEnd,487 (I - DataStart) * sizeof(RawProfData));488 }489 }490 // In binary correlation mode, the CounterPtr is an absolute address of the491 // counter, but it's expected to be relative later when iterating Data.492 IntPtrT CounterOffset = CounterPtr - CountersStart;493 this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset,494 I->FunctionPointer, I->NumCounters);495 }496}497 498template <class IntPtrT>499Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {500 if (this->Ctx->NameSize == 0) {501 return make_error<InstrProfError>(502 instrprof_error::unable_to_correlate_profile,503 "could not find any profile data metadata in object file");504 }505 this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize);506 return Error::success();507}508