261 lines · c
1/*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\2|*3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4|* See https://llvm.org/LICENSE.txt for license information.5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6|*7|*===----------------------------------------------------------------------===*8|* This file defines the API needed for in-process merging of profile data9|* stored in memory buffer.10\*===---------------------------------------------------------------------===*/11 12#include "InstrProfiling.h"13#include "InstrProfilingInternal.h"14#include "InstrProfilingUtil.h"15 16#define INSTR_PROF_VALUE_PROF_DATA17#include "profile/InstrProfData.inc"18 19COMPILER_RT_VISIBILITY20void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);21 22COMPILER_RT_VISIBILITY23uint64_t lprofGetLoadModuleSignature(void) {24 /* A very fast way to compute a module signature. */25 uint64_t Version = __llvm_profile_get_version();26 uint64_t NumCounters = __llvm_profile_get_num_counters(27 __llvm_profile_begin_counters(), __llvm_profile_end_counters());28 uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),29 __llvm_profile_end_data());30 uint64_t NamesSize =31 (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());32 uint64_t NumVnodes =33 (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());34 const __llvm_profile_data *FirstD = __llvm_profile_begin_data();35 36 return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +37 (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +38 __llvm_profile_get_magic();39}40 41#ifdef __GNUC__42#pragma GCC diagnostic push43#pragma GCC diagnostic ignored "-Wcast-qual"44#elif defined(__clang__)45#pragma clang diagnostic push46#pragma clang diagnostic ignored "-Wcast-qual"47#endif48 49/* Returns 1 if profile is not structurally compatible. */50COMPILER_RT_VISIBILITY51int __llvm_profile_check_compatibility(const char *ProfileData,52 uint64_t ProfileSize) {53 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;54 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;55 SrcDataStart =56 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +57 Header->BinaryIdsSize);58 SrcDataEnd = SrcDataStart + Header->NumData;59 60 if (ProfileSize < sizeof(__llvm_profile_header))61 return 1;62 63 /* Check the header first. */64 if (Header->Magic != __llvm_profile_get_magic() ||65 Header->Version != __llvm_profile_get_version() ||66 Header->NumData !=67 __llvm_profile_get_num_data(__llvm_profile_begin_data(),68 __llvm_profile_end_data()) ||69 Header->NumCounters !=70 __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),71 __llvm_profile_end_counters()) ||72 Header->NumBitmapBytes !=73 __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),74 __llvm_profile_end_bitmap()) ||75 Header->NamesSize !=76 __llvm_profile_get_name_size(__llvm_profile_begin_names(),77 __llvm_profile_end_names()) ||78 Header->ValueKindLast != IPVK_Last)79 return 1;80 81 if (ProfileSize <82 sizeof(__llvm_profile_header) + Header->BinaryIdsSize +83 Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +84 Header->NumCounters * __llvm_profile_counter_entry_size() +85 Header->NumBitmapBytes)86 return 1;87 88 for (SrcData = SrcDataStart,89 DstData = (__llvm_profile_data *)__llvm_profile_begin_data();90 SrcData < SrcDataEnd; ++SrcData, ++DstData) {91 if (SrcData->NameRef != DstData->NameRef ||92 SrcData->FuncHash != DstData->FuncHash ||93 SrcData->NumCounters != DstData->NumCounters ||94 SrcData->NumBitmapBytes != DstData->NumBitmapBytes)95 return 1;96 }97 98 /* Matched! */99 return 0;100}101 102static uintptr_t signextIfWin64(void *V) {103#ifdef _WIN64104 return (uintptr_t)(int32_t)(uintptr_t)V;105#else106 return (uintptr_t)V;107#endif108}109 110// Skip names section, vtable profile data section and vtable names section111// for runtime profile merge. To merge runtime addresses from multiple112// profiles collected from the same instrumented binary, the binary should be113// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR114// disabled). In this set-up these three sections remain unchanged.115static uint64_t116getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {117 const uint64_t VTableSectionSize =118 Header->NumVTables * sizeof(VTableProfData);119 const uint64_t PaddingBytesAfterVTableSection =120 __llvm_profile_get_num_padding_bytes(VTableSectionSize);121 const uint64_t VNamesSize = Header->VNamesSize;122 const uint64_t PaddingBytesAfterVNamesSize =123 __llvm_profile_get_num_padding_bytes(VNamesSize);124 return Header->NamesSize +125 __llvm_profile_get_num_padding_bytes(Header->NamesSize) +126 VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +127 PaddingBytesAfterVNamesSize;128}129 130COMPILER_RT_VISIBILITY131int __llvm_profile_merge_from_buffer(const char *ProfileData,132 uint64_t ProfileSize) {133 if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {134 PROF_ERR("%s\n",135 "Temporal profiles do not support profile merging at runtime. "136 "Instead, merge raw profiles using the llvm-profdata tool.");137 return 1;138 }139 140 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;141 uintptr_t CountersDelta = Header->CountersDelta;142 uintptr_t BitmapDelta = Header->BitmapDelta;143 144 __llvm_profile_data *SrcDataStart =145 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +146 Header->BinaryIdsSize);147 __llvm_profile_data *SrcDataEnd = SrcDataStart + Header->NumData;148 uintptr_t SrcCountersStart = (uintptr_t)SrcDataEnd;149 uintptr_t SrcCountersEnd =150 SrcCountersStart +151 Header->NumCounters * __llvm_profile_counter_entry_size();152 uintptr_t SrcBitmapStart =153 SrcCountersEnd +154 __llvm_profile_get_num_padding_bytes(SrcCountersEnd - SrcCountersStart);155 uintptr_t SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;156 uintptr_t SrcValueProfDataStart =157 SrcNameStart + getDistanceFromCounterToValueProf(Header);158 if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)159 return 1;160 161 // Merge counters by iterating the entire counter section when data section is162 // empty due to correlation.163 if (Header->NumData == 0) {164 for (uintptr_t SrcCounter = SrcCountersStart,165 DstCounter = (uintptr_t)__llvm_profile_begin_counters();166 SrcCounter < SrcCountersEnd;) {167 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {168 *(char *)DstCounter &= *(const char *)SrcCounter;169 } else {170 *(uint64_t *)DstCounter += *(const uint64_t *)SrcCounter;171 }172 SrcCounter += __llvm_profile_counter_entry_size();173 DstCounter += __llvm_profile_counter_entry_size();174 }175 return 0;176 }177 178 __llvm_profile_data *SrcData, *DstData;179 uintptr_t SrcValueProfData;180 for (SrcData = SrcDataStart,181 DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),182 SrcValueProfData = SrcValueProfDataStart;183 SrcData < SrcDataEnd; ++SrcData, ++DstData) {184 // For the in-memory destination, CounterPtr is the distance from the start185 // address of the data to the start address of the counter. On WIN64,186 // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign187 // extend CounterPtr to get the original value.188 uintptr_t DstCounters =189 (uintptr_t)DstData + signextIfWin64(DstData->CounterPtr);190 uintptr_t DstBitmap =191 (uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr);192 unsigned NVK = 0;193 194 // SrcData is a serialized representation of the memory image. We need to195 // compute the in-buffer counter offset from the in-memory address distance.196 // The initial CountersDelta is the in-memory address difference197 // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -198 // CountersDelta computes the offset into the in-buffer counter section.199 //200 // On WIN64, CountersDelta is truncated as well, so no need for signext.201 uintptr_t SrcCounters =202 SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);203 // CountersDelta needs to be decreased as we advance to the next data204 // record.205 CountersDelta -= sizeof(*SrcData);206 unsigned NC = SrcData->NumCounters;207 if (NC == 0)208 return 1;209 if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||210 (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)211 return 1;212 for (unsigned I = 0; I < NC; I++) {213 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {214 // A value of zero signifies the function is covered.215 ((char *)DstCounters)[I] &= ((const char *)SrcCounters)[I];216 } else {217 ((uint64_t *)DstCounters)[I] += ((const uint64_t *)SrcCounters)[I];218 }219 }220 221 uintptr_t SrcBitmap =222 SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);223 // BitmapDelta also needs to be decreased as we advance to the next data224 // record.225 BitmapDelta -= sizeof(*SrcData);226 unsigned NB = SrcData->NumBitmapBytes;227 // NumBitmapBytes may legitimately be 0. Just keep going.228 if (NB != 0) {229 if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)230 return 1;231 // Merge Src and Dst Bitmap bytes by simply ORing them together.232 for (unsigned I = 0; I < NB; I++)233 ((char *)DstBitmap)[I] |= ((const char *)SrcBitmap)[I];234 }235 236 /* Now merge value profile data. */237 if (!VPMergeHook)238 continue;239 240 for (unsigned I = 0; I <= IPVK_Last; I++)241 NVK += (SrcData->NumValueSites[I] != 0);242 243 if (!NVK)244 continue;245 246 if (SrcValueProfData >= (uintptr_t)ProfileData + ProfileSize)247 return 1;248 VPMergeHook((ValueProfData *)SrcValueProfData, DstData);249 SrcValueProfData =250 SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;251 }252 253 return 0;254}255 256#ifdef __GNUC__257#pragma GCC diagnostic pop258#elif defined(__clang__)259#pragma clang diagnostic pop260#endif261