275 lines · c
1//===-- Benchmark memory specific tools -------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9// This file complements the `benchmark` header with memory specific tools and10// benchmarking facilities.11 12#ifndef LLVM_LIBC_UTILS_BENCHMARK_MEMORY_BENCHMARK_H13#define LLVM_LIBC_UTILS_BENCHMARK_MEMORY_BENCHMARK_H14 15#include "LibcBenchmark.h"16#include "LibcFunctionPrototypes.h"17#include "MemorySizeDistributions.h"18#include "llvm/ADT/SmallVector.h"19#include "llvm/Support/Alignment.h"20#include "llvm/Support/MathExtras.h"21#include <cstdint>22#include <optional>23#include <random>24 25namespace llvm {26namespace libc_benchmarks {27 28//--------------29// Configuration30//--------------31 32struct StudyConfiguration {33 // One of 'memcpy', 'memset', 'memcmp'.34 // The underlying implementation is always the llvm libc one.35 // e.g. 'memcpy' will test 'LIBC_NAMESPACE::memcpy'36 std::string Function;37 38 // The number of trials to run for this benchmark.39 // If in SweepMode, each individual sizes are measured 'NumTrials' time.40 // i.e 'NumTrials' measurements for 0, 'NumTrials' measurements for 1 ...41 uint32_t NumTrials = 1;42 43 // Toggles between Sweep Mode and Distribution Mode (default).44 // See 'SweepModeMaxSize' and 'SizeDistributionName' below.45 bool IsSweepMode = false;46 47 // Maximum size to use when measuring a ramp of size values (SweepMode).48 // The benchmark measures all sizes from 0 to SweepModeMaxSize.49 // Note: in sweep mode the same size is sampled several times in a row this50 // will allow the processor to learn it and optimize the branching pattern.51 // The resulting measurement is likely to be idealized.52 uint32_t SweepModeMaxSize = 0; // inclusive53 54 // The name of the distribution to be used to randomize the size parameter.55 // This is used when SweepMode is false (default).56 std::string SizeDistributionName;57 58 // This parameter allows to control how the buffers are accessed during59 // benchmark:60 // None : Use a fixed address that is at least cache line aligned,61 // 1 : Use random address,62 // >1 : Use random address aligned to value.63 MaybeAlign AccessAlignment = std::nullopt;64 65 // When Function == 'memcmp', this is the buffers mismatch position.66 // 0 : Buffers always compare equal,67 // >0 : Buffers compare different at byte N-1.68 uint32_t MemcmpMismatchAt = 0;69};70 71struct Runtime {72 // Details about the Host (cpu name, cpu frequency, cache hierarchy).73 HostState Host;74 75 // The framework will populate this value so all data accessed during the76 // benchmark will stay in L1 data cache. This includes bookkeeping data.77 uint32_t BufferSize = 0;78 79 // This is the number of distinct parameters used in a single batch.80 // The framework always tests a batch of randomized parameter to prevent the81 // cpu from learning branching patterns.82 uint32_t BatchParameterCount = 0;83 84 // The benchmark options that were used to perform the measurement.85 // This is decided by the framework.86 BenchmarkOptions BenchmarkOptions;87};88 89//--------90// Results91//--------92 93// The root object containing all the data (configuration and measurements).94struct Study {95 std::string StudyName;96 Runtime Runtime;97 StudyConfiguration Configuration;98 std::vector<Duration> Measurements;99};100 101//------102// Utils103//------104 105// Provides an aligned, dynamically allocated buffer.106class AlignedBuffer {107 char *const Buffer = nullptr;108 size_t Size = 0;109 110public:111 static constexpr size_t Alignment = 512;112 113 explicit AlignedBuffer(size_t Size)114 : Buffer(static_cast<char *>(115 aligned_alloc(Alignment, alignTo(Size, Alignment)))),116 Size(Size) {}117 ~AlignedBuffer() { free(Buffer); }118 119 inline char *operator+(size_t Index) { return Buffer + Index; }120 inline const char *operator+(size_t Index) const { return Buffer + Index; }121 inline char &operator[](size_t Index) { return Buffer[Index]; }122 inline const char &operator[](size_t Index) const { return Buffer[Index]; }123 inline char *begin() { return Buffer; }124 inline char *end() { return Buffer + Size; }125};126 127// Helper to generate random buffer offsets that satisfy the configuration128// constraints.129class OffsetDistribution {130 std::uniform_int_distribution<uint32_t> Distribution;131 uint32_t Factor;132 133public:134 explicit OffsetDistribution(size_t BufferSize, size_t MaxSizeValue,135 MaybeAlign AccessAlignment);136 137 template <class Generator> uint32_t operator()(Generator &G) {138 return Distribution(G) * Factor;139 }140};141 142// Helper to generate random buffer offsets that satisfy the configuration143// constraints. It is specifically designed to benchmark `memcmp` functions144// where we may want the Nth byte to differ.145class MismatchOffsetDistribution {146 std::uniform_int_distribution<size_t> MismatchIndexSelector;147 llvm::SmallVector<uint32_t, 16> MismatchIndices;148 const uint32_t MismatchAt;149 150public:151 explicit MismatchOffsetDistribution(size_t BufferSize, size_t MaxSizeValue,152 size_t MismatchAt);153 154 explicit operator bool() const { return !MismatchIndices.empty(); }155 156 const llvm::SmallVectorImpl<uint32_t> &getMismatchIndices() const {157 return MismatchIndices;158 }159 160 template <class Generator> uint32_t operator()(Generator &G, uint32_t Size) {161 const uint32_t MismatchIndex = MismatchIndices[MismatchIndexSelector(G)];162 // We need to position the offset so that a mismatch occurs at MismatchAt.163 if (Size >= MismatchAt)164 return MismatchIndex - MismatchAt;165 // Size is too small to trigger the mismatch.166 return MismatchIndex - Size - 1;167 }168};169 170/// This structure holds a vector of ParameterType.171/// It makes sure that BufferCount x BufferSize Bytes and the vector of172/// ParameterType can all fit in the L1 cache.173struct ParameterBatch {174 struct ParameterType {175 unsigned OffsetBytes : 16; // max : 16 KiB - 1176 unsigned SizeBytes : 16; // max : 16 KiB - 1177 };178 179 ParameterBatch(size_t BufferCount);180 181 /// Verifies that memory accessed through this parameter is valid.182 void checkValid(const ParameterType &) const;183 184 /// Computes the number of bytes processed during within this batch.185 size_t getBatchBytes() const;186 187 const size_t BufferSize;188 const size_t BatchSize;189 std::vector<ParameterType> Parameters;190};191 192/// Provides source and destination buffers for the Copy operation as well as193/// the associated size distributions.194struct CopySetup : public ParameterBatch {195 CopySetup();196 197 inline static const ArrayRef<MemorySizeDistribution> getDistributions() {198 return getMemcpySizeDistributions();199 }200 201 inline void *Call(ParameterType Parameter, MemcpyFunction Memcpy) {202 return Memcpy(DstBuffer + Parameter.OffsetBytes,203 SrcBuffer + Parameter.OffsetBytes, Parameter.SizeBytes);204 }205 206private:207 AlignedBuffer SrcBuffer;208 AlignedBuffer DstBuffer;209};210 211/// Provides source and destination buffers for the Move operation as well as212/// the associated size distributions.213struct MoveSetup : public ParameterBatch {214 MoveSetup();215 216 inline static const ArrayRef<MemorySizeDistribution> getDistributions() {217 return getMemmoveSizeDistributions();218 }219 220 inline void *Call(ParameterType Parameter, MemmoveFunction Memmove) {221 return Memmove(Buffer + ParameterBatch::BufferSize / 3,222 Buffer + Parameter.OffsetBytes, Parameter.SizeBytes);223 }224 225private:226 AlignedBuffer Buffer;227};228 229/// Provides destination buffer for the Set operation as well as the associated230/// size distributions.231struct SetSetup : public ParameterBatch {232 SetSetup();233 234 inline static const ArrayRef<MemorySizeDistribution> getDistributions() {235 return getMemsetSizeDistributions();236 }237 238 inline void *Call(ParameterType Parameter, MemsetFunction Memset) {239 return Memset(DstBuffer + Parameter.OffsetBytes,240 Parameter.OffsetBytes % 0xFF, Parameter.SizeBytes);241 }242 243 inline void *Call(ParameterType Parameter, BzeroFunction Bzero) {244 Bzero(DstBuffer + Parameter.OffsetBytes, Parameter.SizeBytes);245 return DstBuffer.begin();246 }247 248private:249 AlignedBuffer DstBuffer;250};251 252/// Provides left and right buffers for the Comparison operation as well as the253/// associated size distributions.254struct ComparisonSetup : public ParameterBatch {255 ComparisonSetup();256 257 inline static const ArrayRef<MemorySizeDistribution> getDistributions() {258 return getMemcmpSizeDistributions();259 }260 261 inline int Call(ParameterType Parameter, MemcmpOrBcmpFunction MemcmpOrBcmp) {262 return MemcmpOrBcmp(LhsBuffer + Parameter.OffsetBytes,263 RhsBuffer + Parameter.OffsetBytes, Parameter.SizeBytes);264 }265 266private:267 AlignedBuffer LhsBuffer;268 AlignedBuffer RhsBuffer;269};270 271} // namespace libc_benchmarks272} // namespace llvm273 274#endif // LLVM_LIBC_UTILS_BENCHMARK_MEMORY_BENCHMARK_H275