258 lines · c
1/*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\2|*3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4|* See https://llvm.org/LICENSE.txt for license information.5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6|*7\*===----------------------------------------------------------------------===*/8 9// Note: This is linked into the Darwin kernel, and must remain compatible10// with freestanding compilation. See `darwin_add_builtin_libraries`.11 12#include "InstrProfiling.h"13#include "InstrProfilingInternal.h"14#include "InstrProfilingPort.h"15 16/* When continuous mode is enabled (%c), this parameter is set to 1.17 *18 * This parameter is defined here in InstrProfilingBuffer.o, instead of in19 * InstrProfilingFile.o, to sequester all libc-dependent code in20 * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this21 * layering is violated. */22static int ContinuouslySyncProfile = 0;23 24/* The system page size. Only valid when non-zero. If 0, the page size is25 * unavailable. */26static unsigned PageSize = 0;27 28COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {29 return ContinuouslySyncProfile && PageSize;30}31 32COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {33 ContinuouslySyncProfile = 1;34}35 36COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) {37 ContinuouslySyncProfile = 0;38}39 40COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {41 PageSize = PS;42}43 44COMPILER_RT_VISIBILITY45uint64_t __llvm_profile_get_size_for_buffer(void) {46 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();47 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();48 const char *CountersBegin = __llvm_profile_begin_counters();49 const char *CountersEnd = __llvm_profile_end_counters();50 const char *BitmapBegin = __llvm_profile_begin_bitmap();51 const char *BitmapEnd = __llvm_profile_end_bitmap();52 const char *NamesBegin = __llvm_profile_begin_names();53 const char *NamesEnd = __llvm_profile_end_names();54 const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();55 const VTableProfData *VTableEnd = __llvm_profile_end_vtables();56 const char *VNamesBegin = __llvm_profile_begin_vtabnames();57 const char *VNamesEnd = __llvm_profile_end_vtabnames();58 59 return __llvm_profile_get_size_for_buffer_internal(60 DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd,61 NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);62}63 64// NOTE: Caller should guarantee that `Begin` and `End` specifies a half-open65// interval [Begin, End). Namely, `End` is one-byte past the end of the array.66COMPILER_RT_VISIBILITY67uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,68 const __llvm_profile_data *End) {69 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;70 return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /71 sizeof(__llvm_profile_data);72}73 74COMPILER_RT_VISIBILITY75uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,76 const __llvm_profile_data *End) {77 return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);78}79 80// Counts the number of `VTableProfData` elements within the range of [Begin,81// End). Caller should guarantee that End points to one byte past the inclusive82// range.83// FIXME: Add a compiler-rt test to make sure the number of vtables in the84// raw profile is the same as the number of vtable elements in the instrumented85// binary.86COMPILER_RT_VISIBILITY87uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,88 const VTableProfData *End) {89 // Convert pointers to intptr_t to use integer arithmetic.90 intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin;91 return (EndI - BeginI) / sizeof(VTableProfData);92}93 94COMPILER_RT_VISIBILITY95uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,96 const VTableProfData *End) {97 return (intptr_t)(End) - (intptr_t)(Begin);98}99 100COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {101 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)102 return sizeof(uint8_t);103 return sizeof(uint64_t);104}105 106COMPILER_RT_VISIBILITY107uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) {108 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;109 return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) /110 __llvm_profile_counter_entry_size();111}112 113COMPILER_RT_VISIBILITY114uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) {115 return __llvm_profile_get_num_counters(Begin, End) *116 __llvm_profile_counter_entry_size();117}118 119COMPILER_RT_VISIBILITY120uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,121 const char *End) {122 return (End - Begin);123}124 125COMPILER_RT_VISIBILITY126uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) {127 return End - Begin;128}129 130/// Calculate the number of padding bytes needed to add to \p Offset in order131/// for (\p Offset + Padding) to be page-aligned.132static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {133 uint64_t OffsetModPage = Offset % PageSize;134 if (OffsetModPage > 0)135 return PageSize - OffsetModPage;136 return 0;137}138 139static int needsCounterPadding(void) {140#if defined(__APPLE__)141 return __llvm_profile_is_continuous_mode_enabled();142#else143 return 0;144#endif145}146 147COMPILER_RT_VISIBILITY148int __llvm_profile_get_padding_sizes_for_counters(149 uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,150 uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,151 uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,152 uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames,153 uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) {154 // Counter padding is needed only if continuous mode is enabled.155 if (!needsCounterPadding()) {156 *PaddingBytesBeforeCounters = 0;157 *PaddingBytesAfterCounters =158 __llvm_profile_get_num_padding_bytes(CountersSize);159 *PaddingBytesAfterBitmapBytes =160 __llvm_profile_get_num_padding_bytes(NumBitmapBytes);161 *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);162 if (PaddingBytesAfterVTable != NULL)163 *PaddingBytesAfterVTable =164 __llvm_profile_get_num_padding_bytes(VTableSize);165 if (PaddingBytesAfterVName != NULL)166 *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);167 return 0;168 }169 170 // Value profiling not supported in continuous mode at profile-write time.171 // Return -1 to alert the incompatibility.172 if (VTableSize != 0 || VNameSize != 0)173 return -1;174 175 // In continuous mode, the file offsets for headers and for the start of176 // counter sections need to be page-aligned.177 *PaddingBytesBeforeCounters =178 calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);179 *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);180 *PaddingBytesAfterBitmapBytes =181 calculateBytesNeededToPageAlign(NumBitmapBytes);182 *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);183 // Set these two variables to zero to avoid uninitialized variables184 // even if VTableSize and VNameSize are known to be zero.185 if (PaddingBytesAfterVTable != NULL)186 *PaddingBytesAfterVTable = 0;187 if (PaddingBytesAfterVName != NULL)188 *PaddingBytesAfterVName = 0;189 return 0;190}191 192COMPILER_RT_VISIBILITY193uint64_t __llvm_profile_get_size_for_buffer_internal(194 const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,195 const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin,196 const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd,197 const VTableProfData *VTableBegin, const VTableProfData *VTableEnd,198 const char *VNamesBegin, const char *VNamesEnd) {199 /* Match logic in __llvm_profile_write_buffer(). */200 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);201 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);202 uint64_t CountersSize =203 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);204 const uint64_t NumBitmapBytes =205 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);206 const uint64_t VTableSize =207 __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);208 const uint64_t VNameSize =209 __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);210 211 /* Determine how much padding is needed before/after the counters and after212 * the names. */213 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,214 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,215 PaddingBytesAfterVTable, PaddingBytesAfterVNames;216 __llvm_profile_get_padding_sizes_for_counters(217 DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */,218 0 /* VNameSize */, &PaddingBytesBeforeCounters,219 &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,220 &PaddingBytesAfterNames, &PaddingBytesAfterVTable,221 &PaddingBytesAfterVNames);222 223 return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +224 DataSize + PaddingBytesBeforeCounters + CountersSize +225 PaddingBytesAfterCounters + NumBitmapBytes +226 PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames +227 VTableSize + PaddingBytesAfterVTable + VNameSize +228 PaddingBytesAfterVNames;229}230 231COMPILER_RT_VISIBILITY232void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) {233 BufferWriter->Write = lprofBufferWriter;234 BufferWriter->WriterCtx = Buffer;235}236 237COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) {238 ProfDataWriter BufferWriter;239 initBufferWriter(&BufferWriter, Buffer);240 return lprofWriteData(&BufferWriter, 0, 0);241}242 243COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(244 char *Buffer, const __llvm_profile_data *DataBegin,245 const __llvm_profile_data *DataEnd, const char *CountersBegin,246 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,247 const char *NamesBegin, const char *NamesEnd) {248 ProfDataWriter BufferWriter;249 initBufferWriter(&BufferWriter, Buffer);250 // Set virtual table arguments to NULL since they are not supported yet.251 return lprofWriteDataImpl(252 &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd,253 BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd,254 /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL,255 /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0,256 __llvm_profile_get_version());257}258