381 lines · c
1/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\2|*3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4|* See https://llvm.org/LICENSE.txt for license information.5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6|*7\*===----------------------------------------------------------------------===*/8 9// Note: This is linked into the Darwin kernel, and must remain compatible10// with freestanding compilation. See `darwin_add_builtin_libraries`.11 12#ifdef _MSC_VER13/* For _alloca */14#include <malloc.h>15#endif16#include <string.h>17 18#include "InstrProfiling.h"19#include "InstrProfilingInternal.h"20#include "InstrProfilingPort.h"21 22#define INSTR_PROF_VALUE_PROF_DATA23#include "profile/InstrProfData.inc"24 25COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;26static ProfBufferIO TheBufferIO;27#define VP_BUFFER_SIZE 8 * 102428static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];29static InstrProfValueData VPDataArray[16];30static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);31 32COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;33COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;34 35/* The buffer writer is responsible in keeping writer state36 * across the call.37 */38COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,39 ProfDataIOVec *IOVecs,40 uint32_t NumIOVecs) {41 uint32_t I;42 char **Buffer = (char **)&This->WriterCtx;43 for (I = 0; I < NumIOVecs; I++) {44 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;45 if (IOVecs[I].Data)46 memcpy(*Buffer, IOVecs[I].Data, Length);47 else if (IOVecs[I].UseZeroPadding) {48 /* Allocating the buffer should zero fill. */49 }50 *Buffer += Length;51 }52 return 0;53}54 55static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,56 uint8_t *Buffer, uint32_t BufferSz) {57 BufferIO->FileWriter = FileWriter;58 BufferIO->OwnFileWriter = 0;59 BufferIO->BufferStart = Buffer;60 BufferIO->BufferSz = BufferSz;61 BufferIO->CurOffset = 0;62}63 64COMPILER_RT_VISIBILITY ProfBufferIO *65lprofCreateBufferIO(ProfDataWriter *FileWriter) {66 uint8_t *Buffer = DynamicBufferIOBuffer;67 uint32_t BufferSize = VPBufferSize;68 if (!Buffer) {69 Buffer = &BufferIOBuffer[0];70 BufferSize = sizeof(BufferIOBuffer);71 }72 llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize);73 return &TheBufferIO;74}75 76COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {77 if (BufferIO->OwnFileWriter)78 FreeHook(BufferIO->FileWriter);79 if (DynamicBufferIOBuffer) {80 FreeHook(DynamicBufferIOBuffer);81 DynamicBufferIOBuffer = 0;82 VPBufferSize = 0;83 }84}85 86COMPILER_RT_VISIBILITY int87lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {88 /* Buffer is not large enough, it is time to flush. */89 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {90 if (lprofBufferIOFlush(BufferIO) != 0)91 return -1;92 }93 /* Special case, bypass the buffer completely. */94 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};95 if (Size > BufferIO->BufferSz) {96 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))97 return -1;98 } else {99 /* Write the data to buffer */100 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;101 ProfDataWriter BufferWriter;102 initBufferWriter(&BufferWriter, (char *)Buffer);103 lprofBufferWriter(&BufferWriter, IO, 1);104 BufferIO->CurOffset =105 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;106 }107 return 0;108}109 110COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {111 if (BufferIO->CurOffset) {112 ProfDataIOVec IO[] = {113 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};114 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))115 return -1;116 BufferIO->CurOffset = 0;117 }118 return 0;119}120 121/* Write out value profile data for function specified with \c Data.122 * The implementation does not use the method \c serializeValueProfData123 * which depends on dynamic memory allocation. In this implementation,124 * value profile data is written out to \c BufferIO piecemeal.125 */126static int writeOneValueProfData(ProfBufferIO *BufferIO,127 VPDataReaderType *VPDataReader,128 const __llvm_profile_data *Data) {129 unsigned I, NumValueKinds = 0;130 ValueProfData VPHeader;131 uint8_t *SiteCountArray[IPVK_Last + 1];132 133 for (I = 0; I <= IPVK_Last; I++) {134 if (!Data->NumValueSites[I])135 SiteCountArray[I] = 0;136 else {137 uint32_t Sz =138 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -139 offsetof(ValueProfRecord, SiteCountArray);140 /* Only use alloca for this small byte array to avoid excessive141 * stack growth. */142 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);143 memset(SiteCountArray[I], 0, Sz);144 }145 }146 147 /* If NumValueKinds returned is 0, there is nothing to write, report148 success and return. This should match the raw profile reader's behavior. */149 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))150 return 0;151 152 /* First write the header structure. */153 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();154 VPHeader.NumValueKinds = NumValueKinds;155 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,156 sizeof(ValueProfData)))157 return -1;158 159 /* Make sure nothing else needs to be written before value profile160 * records. */161 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=162 (void *)(&VPHeader + 1))163 return -1;164 165 /* Write out the value profile record for each value kind166 * one by one. */167 for (I = 0; I <= IPVK_Last; I++) {168 uint32_t J;169 ValueProfRecord RecordHeader;170 /* The size of the value prof record header without counting the171 * site count array .*/172 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);173 uint32_t SiteCountArraySize;174 175 if (!Data->NumValueSites[I])176 continue;177 178 /* Write out the record header. */179 RecordHeader.Kind = I;180 RecordHeader.NumValueSites = Data->NumValueSites[I];181 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,182 RecordHeaderSize))183 return -1;184 185 /* Write out the site value count array including padding space. */186 SiteCountArraySize =187 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -188 RecordHeaderSize;189 if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))190 return -1;191 192 /* Write out the value profile data for each value site. */193 for (J = 0; J < Data->NumValueSites[I]; J++) {194 uint32_t NRead, NRemain;195 ValueProfNode *NextStartNode = 0;196 NRemain = VPDataReader->GetNumValueDataForSite(I, J);197 if (!NRemain)198 continue;199 /* Read and write out value data in small chunks till it is done. */200 do {201 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);202 NextStartNode =203 VPDataReader->GetValueData(I, /* ValueKind */204 J, /* Site */205 &VPDataArray[0], NextStartNode, NRead);206 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],207 NRead * sizeof(InstrProfValueData)))208 return -1;209 NRemain -= NRead;210 } while (NRemain != 0);211 }212 }213 /* All done report success. */214 return 0;215}216 217static int writeValueProfData(ProfDataWriter *Writer,218 VPDataReaderType *VPDataReader,219 const __llvm_profile_data *DataBegin,220 const __llvm_profile_data *DataEnd) {221 ProfBufferIO *BufferIO;222 const __llvm_profile_data *DI = 0;223 224 if (!VPDataReader)225 return 0;226 227 BufferIO = lprofCreateBufferIO(Writer);228 229 for (DI = DataBegin; DI < DataEnd; DI++) {230 if (writeOneValueProfData(BufferIO, VPDataReader, DI))231 return -1;232 }233 234 if (lprofBufferIOFlush(BufferIO) != 0)235 return -1;236 lprofDeleteBufferIO(BufferIO);237 238 return 0;239}240 241COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,242 VPDataReaderType *VPDataReader,243 int SkipNameDataWrite) {244 /* Match logic in __llvm_profile_write_buffer(). */245 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();246 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();247 const char *CountersBegin = __llvm_profile_begin_counters();248 const char *CountersEnd = __llvm_profile_end_counters();249 const char *BitmapBegin = __llvm_profile_begin_bitmap();250 const char *BitmapEnd = __llvm_profile_end_bitmap();251 const char *NamesBegin = __llvm_profile_begin_names();252 const char *NamesEnd = __llvm_profile_end_names();253 const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();254 const VTableProfData *VTableEnd = __llvm_profile_end_vtables();255 const char *VNamesBegin = __llvm_profile_begin_vtabnames();256 const char *VNamesEnd = __llvm_profile_end_vtabnames();257 uint64_t Version = __llvm_profile_get_version();258 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,259 CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,260 NamesBegin, NamesEnd, VTableBegin, VTableEnd,261 VNamesBegin, VNamesEnd, SkipNameDataWrite, Version);262}263 264COMPILER_RT_VISIBILITY int lprofWriteDataImpl(265 ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,266 const __llvm_profile_data *DataEnd, const char *CountersBegin,267 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,268 VPDataReaderType *VPDataReader, const char *NamesBegin,269 const char *NamesEnd, const VTableProfData *VTableBegin,270 const VTableProfData *VTableEnd, const char *VNamesBegin,271 const char *VNamesEnd, int SkipNameDataWrite, uint64_t Version) {272 /* Calculate size of sections. */273 const uint64_t DataSectionSize =274 __llvm_profile_get_data_size(DataBegin, DataEnd);275 const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd);276 const uint64_t CountersSectionSize =277 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);278 const uint64_t NumCounters =279 __llvm_profile_get_num_counters(CountersBegin, CountersEnd);280 const uint64_t NumBitmapBytes =281 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);282 const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd);283 const uint64_t NumVTables =284 __llvm_profile_get_num_vtable(VTableBegin, VTableEnd);285 const uint64_t VTableSectionSize =286 __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);287 const uint64_t VNamesSize =288 __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);289 290 /* Create the header. */291 __llvm_profile_header Header;292 293 /* Determine how much padding is needed before/after the counters and after294 * the names. */295 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,296 PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames,297 PaddingBytesAfterVTable, PaddingBytesAfterVNames;298 if (__llvm_profile_get_padding_sizes_for_counters(299 DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,300 VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters,301 &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,302 &PaddingBytesAfterNames, &PaddingBytesAfterVTable,303 &PaddingBytesAfterVNames) == -1)304 return -1;305 306 {307/* Initialize header structure. */308#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;309#include "profile/InstrProfData.inc"310 }311 Header.Version = Version;312 313 /* On WIN64, label differences are truncated 32-bit values. Truncate314 * CountersDelta to match. */315#ifdef _WIN64316 Header.CountersDelta = (uint32_t)Header.CountersDelta;317 Header.BitmapDelta = (uint32_t)Header.BitmapDelta;318#endif319 320 /* The data and names sections are omitted in lightweight mode. */321 if (NumData == 0 && NamesSize == 0) {322 Header.CountersDelta = 0;323 Header.NamesDelta = 0;324 }325 326 /* Write the profile header. */327 ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};328 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))329 return -1;330 331 /* Write the binary id lengths and data. */332 if (__llvm_write_binary_ids(Writer) == -1)333 return -1;334 335 /* Write the profile data. */336 ProfDataIOVec IOVecData[] = {337 {DataBegin, sizeof(uint8_t), DataSectionSize, 0},338 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},339 {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},340 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},341 {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},342 {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},343 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},344 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1},345 {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0},346 {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1},347 {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0},348 {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}};349 if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))350 return -1;351 352 /* Value profiling is not yet supported in continuous mode and profile353 * correlation mode. */354 if (__llvm_profile_is_continuous_mode_enabled() ||355 (NumData == 0 && NamesSize == 0))356 return 0;357 358 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);359}360 361/*362 * Write binary id length and then its data, because binary id does not363 * have a fixed length.364 */365COMPILER_RT_VISIBILITY366int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,367 const uint8_t *BinaryIdData,368 uint64_t BinaryIdPadding) {369 ProfDataIOVec BinaryIdIOVec[] = {370 {&BinaryIdLen, sizeof(uint64_t), 1, 0},371 {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0},372 {NULL, sizeof(uint8_t), BinaryIdPadding, 1},373 };374 if (Writer->Write(Writer, BinaryIdIOVec,375 sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))376 return -1;377 378 /* Successfully wrote binary id, report success. */379 return 0;380}381