1002 lines · plain
1/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\2|*3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4|* See https://llvm.org/LICENSE.txt for license information.5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6|*7\*===----------------------------------------------------------------------===*/8/*9 * This is the main file that defines all the data structure, signature,10 * constant literals that are shared across profiling runtime library,11 * compiler (instrumentation), and host tools (reader/writer). The entities12 * defined in this file affect the profile runtime ABI, the raw profile format,13 * or both.14 *15 * The file has two identical copies. The primary copy lives in LLVM and16 * the other one sits in compiler-rt/lib/profile directory. To make changes17 * in this file, first modify the primary copy and copy it over to compiler-rt.18 * Testing of any change in this file can start only after the two copies are19 * synced up.20 *21 * The first part of the file includes macros that defines types, names, and22 * initializers for the member fields of the core data structures. The field23 * declarations for one structure is enabled by defining the field activation24 * macro associated with that structure. Only one field activation record25 * can be defined at one time and the rest definitions will be filtered out by26 * the preprocessor.27 *28 * Examples of how the template is used to instantiate structure definition:29 * 1. To declare a structure:30 *31 * struct ProfData {32 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \33 * Type Name;34 * #include "llvm/ProfileData/InstrProfData.inc"35 * };36 *37 * 2. To construct LLVM type arrays for the struct type:38 *39 * Type *DataTypes[] = {40 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \41 * LLVMType,42 * #include "llvm/ProfileData/InstrProfData.inc"43 * };44 *45 * 4. To construct constant array for the initializers:46 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \47 * Initializer,48 * Constant *ConstantVals[] = {49 * #include "llvm/ProfileData/InstrProfData.inc"50 * };51 *52 *53 * The second part of the file includes definitions all other entities that54 * are related to runtime ABI and format. When no field activation macro is55 * defined, this file can be included to introduce the definitions.56 *57\*===----------------------------------------------------------------------===*/58 59/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in60 * the compiler runtime. */61#ifndef INSTR_PROF_VISIBILITY62#define INSTR_PROF_VISIBILITY63#endif64 65// clang-format off:consider re-enabling clang-format if auto-formatted C macros66// are readable (e.g., after `issue #82426` is fixed)67/* INSTR_PROF_DATA start. */68/* Definition of member fields of the per-function control structure. */69#ifndef INSTR_PROF_DATA70#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)71#else72#define INSTR_PROF_DATA_DEFINED73#endif74INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \75 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \76 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))77INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \78 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \79 Inc->getHash()->getZExtValue()))80INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)81INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)82/* This is used to map function pointers for the indirect call targets to83 * function name hashes during the conversion from raw to merged profile84 * data.85 */86INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \87 FunctionAddr)88INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \89 ValuesPtrExpr)90INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \91 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))92INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \93 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \94INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \95 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))96#undef INSTR_PROF_DATA97/* INSTR_PROF_DATA end. */98 99/* For a virtual table object, record the name hash to associate profiled100 * addresses with global variables, and record {starting address, size in bytes}101 * to map the profiled virtual table (which usually have an offset from the102 * starting address) back to a virtual table object. */103#ifndef INSTR_PROF_VTABLE_DATA104#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)105#else106#define INSTR_PROF_VTABLE_DATA_DEFINED107#endif108INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \109 VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \110 IndexedInstrProf::ComputeHash(PGOVTableName)))111INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \112 VTablePointer, VTableAddr)113INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \114 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \115 VTableSizeVal))116#undef INSTR_PROF_VTABLE_DATA117/* INSTR_PROF_VTABLE_DATA end. */118 119/* This is an internal data structure used by value profiler. It120 * is defined here to allow serialization code sharing by LLVM121 * to be used in unit test.122 *123 * typedef struct ValueProfNode {124 * // InstrProfValueData VData;125 * uint64_t Value;126 * uint64_t Count;127 * struct ValueProfNode *Next;128 * } ValueProfNode;129 */130/* INSTR_PROF_VALUE_NODE start. */131#ifndef INSTR_PROF_VALUE_NODE132#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)133#else134#define INSTR_PROF_DATA_DEFINED135#endif136INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \137 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))138INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \139 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))140INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \141 ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0))142#undef INSTR_PROF_VALUE_NODE143/* INSTR_PROF_VALUE_NODE end. */144 145/* INSTR_PROF_RAW_HEADER start */146/* Definition of member fields of the raw profile header data structure. */147/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating148 raw profile format. */149#ifndef INSTR_PROF_RAW_HEADER150#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)151#else152#define INSTR_PROF_DATA_DEFINED153#endif154INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())155INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())156INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))157INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)158INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)159INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)160INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)161INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)162INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)163INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)164INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,165 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)166INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,167 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)168INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)169INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)170INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)171INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)172#undef INSTR_PROF_RAW_HEADER173/* INSTR_PROF_RAW_HEADER end */174 175/* VALUE_PROF_FUNC_PARAM start */176/* Definition of parameter types of the runtime API used to do value profiling177 * for a given value site.178 */179#ifndef VALUE_PROF_FUNC_PARAM180#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)181#define INSTR_PROF_COMMA182#else183#define INSTR_PROF_DATA_DEFINED184#define INSTR_PROF_COMMA ,185#endif186VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \187 INSTR_PROF_COMMA188VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA189VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))190#undef VALUE_PROF_FUNC_PARAM191#undef INSTR_PROF_COMMA192/* VALUE_PROF_FUNC_PARAM end */193 194/* VALUE_PROF_KIND start */195#ifndef VALUE_PROF_KIND196#define VALUE_PROF_KIND(Enumerator, Value, Descr)197#else198#define INSTR_PROF_DATA_DEFINED199#endif200/* For indirect function call value profiling, the addresses of the target201 * functions are profiled by the instrumented code. The target addresses are202 * written in the raw profile data and converted to target function name's MD5203 * hash by the profile reader during deserialization. Typically, this happens204 * when the raw profile data is read during profile merging.205 *206 * For this remapping the ProfData is used. ProfData contains both the function207 * name hash and the function address.208 */209VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")210/* For memory intrinsic functions size profiling. */211VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")212/* For virtual table address profiling, the address point of the virtual table213 * (i.e., the address contained in objects pointing to a virtual table) are214 * profiled. Note this may not be the address of the per C++ class virtual table215 * object (e.g., there might be an offset).216 *217 * The profiled addresses are stored in raw profile, together with the following218 * two types of information.219 * 1. The (starting and ending) addresses of per C++ class virtual table objects.220 * 2. The (compressed) virtual table object names.221 * RawInstrProfReader converts profiled virtual table addresses to virtual table222 * objects' MD5 hash.223 */224VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable")225/* These two kinds must be the last to be226 * declared. This is to make sure the string227 * array created with the template can be228 * indexed with the kind value.229 */230VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")231VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")232 233#undef VALUE_PROF_KIND234/* VALUE_PROF_KIND end */235 236#undef COVMAP_V2_OR_V3237#ifdef COVMAP_V2238#define COVMAP_V2_OR_V3239#endif240#ifdef COVMAP_V3241#define COVMAP_V2_OR_V3242#endif243 244/* COVMAP_FUNC_RECORD start */245/* Definition of member fields of the function record structure in coverage246 * map.247 */248#ifndef COVMAP_FUNC_RECORD249#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)250#else251#define INSTR_PROF_DATA_DEFINED252#endif253#ifdef COVMAP_V1254COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \255 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \256 llvm::PointerType::getUnqual(Ctx)))257COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \258 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \259 NameValue.size()))260#endif261#ifdef COVMAP_V2_OR_V3262COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \263 llvm::ConstantInt::get( \264 llvm::Type::getInt64Ty(Ctx), NameHash))265#endif266COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \267 llvm::ConstantInt::get( \268 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))269COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \270 llvm::ConstantInt::get( \271 llvm::Type::getInt64Ty(Ctx), FuncHash))272#ifdef COVMAP_V3273COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \274 llvm::ConstantInt::get( \275 llvm::Type::getInt64Ty(Ctx), FilenamesRef))276COVMAP_FUNC_RECORD(const char, \277 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \278 CoverageMapping.size()), \279 CoverageMapping,280 llvm::ConstantDataArray::getRaw( \281 CoverageMapping, CoverageMapping.size(), \282 llvm::Type::getInt8Ty(Ctx)))283#endif284#undef COVMAP_FUNC_RECORD285/* COVMAP_FUNC_RECORD end. */286 287/* COVMAP_HEADER start */288/* Definition of member fields of coverage map header.289 */290#ifndef COVMAP_HEADER291#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)292#else293#define INSTR_PROF_DATA_DEFINED294#endif295COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \296 llvm::ConstantInt::get(Int32Ty, NRecords))297COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \298 llvm::ConstantInt::get(Int32Ty, FilenamesSize))299COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \300 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))301COVMAP_HEADER(uint32_t, Int32Ty, Version, \302 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))303#undef COVMAP_HEADER304/* COVMAP_HEADER end. */305 306/* COVINIT_FUNC start */307#ifndef COVINIT_FUNC308#define COVINIT_FUNC(Type, LLVMType, Name, Initializer)309#else310#define INSTR_PROF_DATA_DEFINED311#endif312COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \313 WriteoutF)314COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \315 ResetF)316#undef COVINIT_FUNC317/* COVINIT_FUNC end */318 319#ifdef INSTR_PROF_SECT_ENTRY320#define INSTR_PROF_DATA_DEFINED321INSTR_PROF_SECT_ENTRY(IPSK_data, \322 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \323 INSTR_PROF_DATA_COFF, "__DATA,")324INSTR_PROF_SECT_ENTRY(IPSK_cnts, \325 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \326 INSTR_PROF_CNTS_COFF, "__DATA,")327INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \328 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \329 INSTR_PROF_BITS_COFF, "__DATA,")330INSTR_PROF_SECT_ENTRY(IPSK_name, \331 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \332 INSTR_PROF_NAME_COFF, "__DATA,")333INSTR_PROF_SECT_ENTRY(IPSK_vname, \334 INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \335 INSTR_PROF_VNAME_COFF, "__DATA,")336INSTR_PROF_SECT_ENTRY(IPSK_vals, \337 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \338 INSTR_PROF_VALS_COFF, "__DATA,")339INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \340 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \341 INSTR_PROF_VNODES_COFF, "__DATA,")342INSTR_PROF_SECT_ENTRY(IPSK_vtab, \343 INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \344 INSTR_PROF_VTAB_COFF, "__DATA,")345INSTR_PROF_SECT_ENTRY(IPSK_covmap, \346 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \347 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")348INSTR_PROF_SECT_ENTRY(IPSK_covfun, \349 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \350 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")351INSTR_PROF_SECT_ENTRY(IPSK_covdata, \352 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \353 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")354INSTR_PROF_SECT_ENTRY(IPSK_covname, \355 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \356 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")357INSTR_PROF_SECT_ENTRY(IPSK_covinit, \358 INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \359 INSTR_PROF_COVINIT_COFF, "__LLVM_COV,")360 361#undef INSTR_PROF_SECT_ENTRY362#endif363 364 365#ifdef INSTR_PROF_VALUE_PROF_DATA366#define INSTR_PROF_DATA_DEFINED367 368#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255369/*!370 * This is the header of the data structure that defines the on-disk371 * layout of the value profile data of a particular kind for one function.372 */373typedef struct ValueProfRecord {374 /* The kind of the value profile record. */375 uint32_t Kind;376 /*377 * The number of value profile sites. It is guaranteed to be non-zero;378 * otherwise the record for this kind won't be emitted.379 */380 uint32_t NumValueSites;381 /*382 * The first element of the array that stores the number of profiled383 * values for each value site. The size of the array is NumValueSites.384 * Since NumValueSites is greater than zero, there is at least one385 * element in the array.386 */387 uint8_t SiteCountArray[1];388 389 /*390 * The fake declaration is for documentation purpose only.391 * Align the start of next field to be on 8 byte boundaries.392 uint8_t Padding[X];393 */394 395 /* The array of value profile data. The size of the array is the sum396 * of all elements in SiteCountArray[].397 InstrProfValueData ValueData[];398 */399 400#ifdef __cplusplus401 /*!402 * Return the number of value sites.403 */404 uint32_t getNumValueSites() const { return NumValueSites; }405 /*!406 * Read data from this record and save it to Record.407 */408 LLVM_ABI void deserializeTo(InstrProfRecord &Record,409 InstrProfSymtab *SymTab);410 /*411 * In-place byte swap:412 * Do byte swap for this instance. \c Old is the original order before413 * the swap, and \c New is the New byte order.414 */415 LLVM_ABI void swapBytes(llvm::endianness Old, llvm::endianness New);416#endif417} ValueProfRecord;418 419/*!420 * Per-function header/control data structure for value profiling421 * data in indexed format.422 */423typedef struct ValueProfData {424 /*425 * Total size in bytes including this field. It must be a multiple426 * of sizeof(uint64_t).427 */428 uint32_t TotalSize;429 /*430 *The number of value profile kinds that has value profile data.431 * In this implementation, a value profile kind is considered to432 * have profile data if the number of value profile sites for the433 * kind is not zero. More aggressively, the implementation can434 * choose to check the actual data value: if none of the value sites435 * has any profiled values, the kind can be skipped.436 */437 uint32_t NumValueKinds;438 439 /*440 * Following are a sequence of variable length records. The prefix/header441 * of each record is defined by ValueProfRecord type. The number of442 * records is NumValueKinds.443 * ValueProfRecord Record_1;444 * ValueProfRecord Record_N;445 */446 447#if __cplusplus448 /*!449 * Return the total size in bytes of the on-disk value profile data450 * given the data stored in Record.451 */452 LLVM_ABI static uint32_t getSize(const InstrProfRecord &Record);453 /*!454 * Return a pointer to \c ValueProfData instance ready to be streamed.455 */456 LLVM_ABI static std::unique_ptr<ValueProfData>457 serializeFrom(const InstrProfRecord &Record);458 /*!459 * Check the integrity of the record.460 */461 LLVM_ABI Error checkIntegrity();462 /*!463 * Return a pointer to \c ValueProfileData instance ready to be read.464 * All data in the instance are properly byte swapped. The input465 * data is assumed to be in little endian order.466 */467 LLVM_ABI static Expected<std::unique_ptr<ValueProfData>>468 getValueProfData(const unsigned char *SrcBuffer,469 const unsigned char *const SrcBufferEnd,470 llvm::endianness SrcDataEndianness);471 /*!472 * Swap byte order from \c Endianness order to host byte order.473 */474 LLVM_ABI void swapBytesToHost(llvm::endianness Endianness);475 /*!476 * Swap byte order from host byte order to \c Endianness order.477 */478 LLVM_ABI void swapBytesFromHost(llvm::endianness Endianness);479 /*!480 * Return the total size of \c ValueProfileData.481 */482 LLVM_ABI uint32_t getSize() const { return TotalSize; }483 /*!484 * Read data from this data and save it to \c Record.485 */486 LLVM_ABI void deserializeTo(InstrProfRecord &Record,487 InstrProfSymtab *SymTab);488 void operator delete(void *ptr) { ::operator delete(ptr); }489#endif490} ValueProfData;491 492/*493 * The closure is designed to abstract away two types of value profile data:494 * - InstrProfRecord which is the primary data structure used to495 * represent profile data in host tools (reader, writer, and profile-use)496 * - value profile runtime data structure suitable to be used by C497 * runtime library.498 *499 * Both sources of data need to serialize to disk/memory-buffer in common500 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler501 * writer to share the same format and code with indexed profile writer.502 *503 * For documentation of the member methods below, refer to corresponding methods504 * in class InstrProfRecord.505 */506typedef struct ValueProfRecordClosure {507 const void *Record;508 uint32_t (*GetNumValueKinds)(const void *Record);509 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);510 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);511 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);512 513 /*514 * After extracting the value profile data from the value profile record,515 * this method is used to map the in-memory value to on-disk value. If516 * the method is null, value will be written out untranslated.517 */518 uint64_t (*RemapValueData)(uint32_t, uint64_t Value);519 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,520 uint32_t S);521 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);522} ValueProfRecordClosure;523 524INSTR_PROF_VISIBILITY ValueProfRecord *525getFirstValueProfRecord(ValueProfData *VPD);526INSTR_PROF_VISIBILITY ValueProfRecord *527getValueProfRecordNext(ValueProfRecord *VPR);528INSTR_PROF_VISIBILITY InstrProfValueData *529getValueProfRecordValueData(ValueProfRecord *VPR);530INSTR_PROF_VISIBILITY uint32_t531getValueProfRecordHeaderSize(uint32_t NumValueSites);532 533#undef INSTR_PROF_VALUE_PROF_DATA534#endif /* INSTR_PROF_VALUE_PROF_DATA */535 536 537#ifdef INSTR_PROF_COMMON_API_IMPL538#define INSTR_PROF_DATA_DEFINED539#ifdef __cplusplus540#define INSTR_PROF_INLINE inline541#define INSTR_PROF_NULLPTR nullptr542#else543#define INSTR_PROF_INLINE544#define INSTR_PROF_NULLPTR NULL545#endif546 547#ifndef offsetof548#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)549#endif550 551// clang-format on552 553/*!554 * Return the \c ValueProfRecord header size including the555 * padding bytes.556 */557INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t558getValueProfRecordHeaderSize(uint32_t NumValueSites) {559 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +560 sizeof(uint8_t) * NumValueSites;561 /* Round the size to multiple of 8 bytes. */562 Size = (Size + 7) & ~7;563 return Size;564}565 566/*!567 * Return the total size of the value profile record including the568 * header and the value data.569 */570INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t571getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) {572 return getValueProfRecordHeaderSize(NumValueSites) +573 sizeof(InstrProfValueData) * NumValueData;574}575 576/*!577 * Return the pointer to the start of value data array.578 */579INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *580getValueProfRecordValueData(ValueProfRecord *This) {581 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(582 This->NumValueSites));583}584 585/*!586 * Return the total number of value data for \c This record.587 */588INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t589getValueProfRecordNumValueData(ValueProfRecord *This) {590 uint32_t NumValueData = 0;591 uint32_t I;592 for (I = 0; I < This->NumValueSites; I++)593 NumValueData += This->SiteCountArray[I];594 return NumValueData;595}596 597/*!598 * Use this method to advance to the next \c This \c ValueProfRecord.599 */600INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *601getValueProfRecordNext(ValueProfRecord *This) {602 uint32_t NumValueData = getValueProfRecordNumValueData(This);603 return (ValueProfRecord *)((char *)This +604 getValueProfRecordSize(This->NumValueSites,605 NumValueData));606}607 608/*!609 * Return the first \c ValueProfRecord instance.610 */611INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *612getFirstValueProfRecord(ValueProfData *This) {613 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));614}615 616/* Closure based interfaces. */617 618/*!619 * Return the total size in bytes of the on-disk value profile data620 * given the data stored in Record.621 */622INSTR_PROF_VISIBILITY uint32_t623getValueProfDataSize(ValueProfRecordClosure *Closure) {624 uint32_t Kind;625 uint32_t TotalSize = sizeof(ValueProfData);626 const void *Record = Closure->Record;627 628 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {629 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);630 if (!NumValueSites)631 continue;632 TotalSize += getValueProfRecordSize(NumValueSites,633 Closure->GetNumValueData(Record, Kind));634 }635 return TotalSize;636}637 638/*!639 * Extract value profile data of a function for the profile kind \c ValueKind640 * from the \c Closure and serialize the data into \c This record instance.641 */642INSTR_PROF_VISIBILITY void643serializeValueProfRecordFrom(ValueProfRecord *This,644 ValueProfRecordClosure *Closure,645 uint32_t ValueKind, uint32_t NumValueSites) {646 uint32_t S;647 const void *Record = Closure->Record;648 This->Kind = ValueKind;649 This->NumValueSites = NumValueSites;650 InstrProfValueData *DstVD = getValueProfRecordValueData(This);651 652 for (S = 0; S < NumValueSites; S++) {653 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);654 This->SiteCountArray[S] = ND;655 Closure->GetValueForSite(Record, DstVD, ValueKind, S);656 DstVD += ND;657 }658}659 660/*!661 * Extract value profile data of a function from the \c Closure662 * and serialize the data into \c DstData if it is not NULL or heap663 * memory allocated by the \c Closure's allocator method. If \c664 * DstData is not null, the caller is expected to set the TotalSize665 * in DstData.666 */667INSTR_PROF_VISIBILITY ValueProfData *668serializeValueProfDataFrom(ValueProfRecordClosure *Closure,669 ValueProfData *DstData) {670 uint32_t Kind;671 uint32_t TotalSize =672 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);673 674 ValueProfData *VPD =675 DstData ? DstData : Closure->AllocValueProfData(TotalSize);676 677 VPD->TotalSize = TotalSize;678 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);679 ValueProfRecord *VR = getFirstValueProfRecord(VPD);680 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {681 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);682 if (!NumValueSites)683 continue;684 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);685 VR = getValueProfRecordNext(VR);686 }687 return VPD;688}689 690#undef INSTR_PROF_COMMON_API_IMPL691#endif /* INSTR_PROF_COMMON_API_IMPL */692 693/*============================================================================*/694 695// clang-format off:consider re-enabling clang-format if auto-formatted C macros696// are readable (e.g., after `issue #82426` is fixed)697#ifndef INSTR_PROF_DATA_DEFINED698 699#ifndef INSTR_PROF_DATA_INC700#define INSTR_PROF_DATA_INC701 702/* Helper macros. */703#define INSTR_PROF_SIMPLE_QUOTE(x) #x704#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)705#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y706#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)707 708/* Magic number to detect file format and endianness.709 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,710 * so that utilities, like strings, don't grab it as a string. 129 is also711 * invalid UTF-8, and high enough to be interesting.712 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"713 * for 32-bit platforms.714 */715#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \716 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \717 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129718#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \719 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \720 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129721 722/* Raw profile format version (start from 1). */723#define INSTR_PROF_RAW_VERSION 10724/* Indexed profile format version (start from 1). */725#define INSTR_PROF_INDEX_VERSION 13726/* Coverage mapping format version (start from 0). */727#define INSTR_PROF_COVMAP_VERSION 6728 729/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the730 * version for other variants of profile. We set the 8th most significant bit731 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation732 * generated profile, and 0 if this is a Clang FE generated profile.733 * 1 in bit 57 indicates there are context-sensitive records in the profile.734 * The 54th bit indicates whether to always instrument loop entry blocks.735 * The 58th bit indicates whether to always instrument function entry blocks.736 * The 59th bit indicates whether to use debug info to correlate profiles.737 * The 60th bit indicates single byte coverage instrumentation.738 * The 61st bit indicates function entry instrumentation only.739 * The 62nd bit indicates whether memory profile information is present.740 * The 63rd bit indicates if this is a temporal profile.741 */742#define VARIANT_MASKS_ALL 0xffffffff00000000ULL743#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)744#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)745#define VARIANT_MASK_IR_PROF (0x1ULL << 56)746#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)747#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)748#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)749#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)750#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)751#define VARIANT_MASK_MEMPROF (0x1ULL << 62)752#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)753#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version754#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime755#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias756#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias757#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp758#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling759 760/* The variable that holds the name of the profile data761 * specified via command line. */762#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename763 764/* section name strings common to all targets other765 than WIN32 */766#define INSTR_PROF_DATA_COMMON __llvm_prf_data767#define INSTR_PROF_NAME_COMMON __llvm_prf_names768#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns769#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts770#define INSTR_PROF_BITS_COMMON __llvm_prf_bits771#define INSTR_PROF_VALS_COMMON __llvm_prf_vals772#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds773#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab774#define INSTR_PROF_COVMAP_COMMON __llvm_covmap775#define INSTR_PROF_COVFUN_COMMON __llvm_covfun776#define INSTR_PROF_COVDATA_COMMON __llvm_covdata777#define INSTR_PROF_COVNAME_COMMON __llvm_covnames778#define INSTR_PROF_COVINIT_COMMON __llvm_covinit779 780/* Windows section names. Because these section names contain dollar characters,781 * they must be quoted.782 */783#define INSTR_PROF_DATA_COFF ".lprfd$M"784#define INSTR_PROF_NAME_COFF ".lprfn$M"785#define INSTR_PROF_VNAME_COFF ".lprfvn$M"786#define INSTR_PROF_CNTS_COFF ".lprfc$M"787#define INSTR_PROF_BITS_COFF ".lprfb$M"788#define INSTR_PROF_VALS_COFF ".lprfv$M"789#define INSTR_PROF_VNODES_COFF ".lprfnd$M"790#define INSTR_PROF_VTAB_COFF ".lprfvt$M"791#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"792#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"793/* Since cov data and cov names sections are not allocated, we don't need to794 * access them at runtime.795 */796#define INSTR_PROF_COVDATA_COFF ".lcovd"797#define INSTR_PROF_COVNAME_COFF ".lcovn"798 799// FIXME: Placeholder for Windows. Windows currently does not initialize800// the GCOV functions in the runtime.801#define INSTR_PROF_COVINIT_COFF ".lcovd$M"802 803#ifdef _WIN32804/* Runtime section names and name strings. */805#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF806#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF807#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF808#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF809#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF810#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF811/* Array of pointers. Each pointer points to a list812 * of value nodes associated with one value site.813 */814#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF815/* Value profile nodes section. */816#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF817#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF818#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF819#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF820#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF821#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF822#else823/* Runtime section names and name strings. */824#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)825#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)826#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)827#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)828#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON)829#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON)830/* Array of pointers. Each pointer points to a list831 * of value nodes associated with one value site.832 */833#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)834/* Value profile nodes section. */835#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)836#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)837#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)838#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)839#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)840#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON)841#endif842 843/* Macros to define start/stop section symbol for a given844 * section on Linux. For instance845 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will846 * expand to __start___llvm_prof_data847 */848#define INSTR_PROF_SECT_START(Sect) \849 INSTR_PROF_CONCAT(__start_,Sect)850#define INSTR_PROF_SECT_STOP(Sect) \851 INSTR_PROF_CONCAT(__stop_,Sect)852 853/* Value Profiling API linkage name. */854#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target855#define INSTR_PROF_VALUE_PROF_FUNC_STR \856 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)857#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop858#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \859 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)860 861/* InstrProfile per-function control data alignment. */862#define INSTR_PROF_DATA_ALIGNMENT 8863 864/* The data structure that represents a tracked value by the865 * value profiler.866 */867typedef struct InstrProfValueData {868 /* Profiled value. */869 uint64_t Value;870 /* Number of times the value appears in the training run. */871 uint64_t Count;872} InstrProfValueData;873 874#endif /* INSTR_PROF_DATA_INC */875 876#else877#undef INSTR_PROF_DATA_DEFINED878#endif879 880#undef COVMAP_V2_OR_V3881 882#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API883 884#ifdef __cplusplus885#define INSTR_PROF_INLINE inline886#else887#define INSTR_PROF_INLINE888#endif889 890/* The value range buckets (22 buckets) for the memop size value profiling looks891 * like:892 *893 * [0, 0]894 * [1, 1]895 * [2, 2]896 * [3, 3]897 * [4, 4]898 * [5, 5]899 * [6, 6]900 * [7, 7]901 * [8, 8]902 * [9, 15]903 * [16, 16]904 * [17, 31]905 * [32, 32]906 * [33, 63]907 * [64, 64]908 * [65, 127]909 * [128, 128]910 * [129, 255]911 * [256, 256]912 * [257, 511]913 * [512, 512]914 * [513, UINT64_MAX]915 *916 * Each range has a 'representative value' which is the lower end value of the917 * range and used to store in the runtime profile data records and the VP918 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].919 */920#define INSTR_PROF_NUM_BUCKETS 22921 922/*923 * Clz and Popcount. This code was copied from924 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and925 * llvm/include/llvm/Support/MathExtras.h.926 */927#if defined(_MSC_VER) && !defined(__clang__)928 929#include <intrin.h>930INSTR_PROF_VISIBILITY INSTR_PROF_INLINE931int InstProfClzll(unsigned long long X) {932 unsigned long LeadZeroIdx = 0;933#if !defined(_M_ARM64) && !defined(_M_X64)934 // Scan the high 32 bits.935 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))936 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset937 // from the MSB.938 // Scan the low 32 bits.939 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))940 return (int)(63 - LeadZeroIdx);941#else942 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;943#endif944 return 64;945}946INSTR_PROF_VISIBILITY INSTR_PROF_INLINE947int InstProfPopcountll(unsigned long long X) {948 // This code originates from https://reviews.llvm.org/rG30626254510f.949 unsigned long long v = X;950 v = v - ((v >> 1) & 0x5555555555555555ULL);951 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);952 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;953 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);954}955 956#else957 958INSTR_PROF_VISIBILITY INSTR_PROF_INLINE959int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }960INSTR_PROF_VISIBILITY INSTR_PROF_INLINE961int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }962 963#endif /* defined(_MSC_VER) && !defined(__clang__) */964 965// clang-format on966 967/* Map an (observed) memop size value to the representative value of its range.968 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */969INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t970InstrProfGetRangeRepValue(uint64_t Value) {971 if (Value <= 8)972 // The first ranges are individually tracked. Use the value as is.973 return Value;974 else if (Value >= 513)975 // The last range is mapped to its lowest value.976 return 513;977 else if (InstProfPopcountll(Value) == 1)978 // If it's a power of two, use it as is.979 return Value;980 else981 // Otherwise, take to the previous power of two + 1.982 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;983}984 985/* Return true if the range that an (observed) memop size value belongs to has986 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->987 * false, 64 -> true, 100 -> false, 513 -> false. */988INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned989InstrProfIsSingleValRange(uint64_t Value) {990 if (Value <= 8)991 // The first ranges are individually tracked.992 return 1;993 else if (InstProfPopcountll(Value) == 1)994 // If it's a power of two, there's only one value.995 return 1;996 else997 // Otherwise, there's more than one value in the range.998 return 0;999}1000 1001#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */1002