283 lines · plain
1// Copyright 2021 Google Inc. All rights reserved.2//3// Licensed under the Apache License, Version 2.0 (the "License");4// you may not use this file except in compliance with the License.5// You may obtain a copy of the License at6//7// http://www.apache.org/licenses/LICENSE-2.08//9// Unless required by applicable law or agreed to in writing, software10// distributed under the License is distributed on an "AS IS" BASIS,11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12// See the License for the specific language governing permissions and13// limitations under the License.14 15#include "perf_counters.h"16 17#include <cstring>18#include <memory>19#include <vector>20 21#if defined HAVE_LIBPFM22#include "perfmon/pfmlib.h"23#include "perfmon/pfmlib_perf_event.h"24#endif25 26namespace benchmark {27namespace internal {28 29constexpr size_t PerfCounterValues::kMaxCounters;30 31#if defined HAVE_LIBPFM32 33size_t PerfCounterValues::Read(const std::vector<int>& leaders) {34 // Create a pointer for multiple reads35 const size_t bufsize = values_.size() * sizeof(values_[0]);36 char* ptr = reinterpret_cast<char*>(values_.data());37 size_t size = bufsize;38 for (int lead : leaders) {39 auto read_bytes = ::read(lead, ptr, size);40 if (read_bytes >= ssize_t(sizeof(uint64_t))) {41 // Actual data bytes are all bytes minus initial padding42 std::size_t data_bytes = read_bytes - sizeof(uint64_t);43 // This should be very cheap since it's in hot cache44 std::memmove(ptr, ptr + sizeof(uint64_t), data_bytes);45 // Increment our counters46 ptr += data_bytes;47 size -= data_bytes;48 } else {49 int err = errno;50 GetErrorLogInstance() << "Error reading lead " << lead << " errno:" << err51 << " " << ::strerror(err) << "\n";52 return 0;53 }54 }55 return (bufsize - size) / sizeof(uint64_t);56}57 58const bool PerfCounters::kSupported = true;59 60// Initializes libpfm only on the first call. Returns whether that single61// initialization was successful.62bool PerfCounters::Initialize() {63 // Function-scope static gets initialized only once on first call.64 static const bool success = []() {65 return pfm_initialize() == PFM_SUCCESS;66 }();67 return success;68}69 70bool PerfCounters::IsCounterSupported(const std::string& name) {71 Initialize();72 perf_event_attr_t attr;73 std::memset(&attr, 0, sizeof(attr));74 pfm_perf_encode_arg_t arg;75 std::memset(&arg, 0, sizeof(arg));76 arg.attr = &attr;77 const int mode = PFM_PLM3; // user mode only78 int ret = pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT_EXT,79 &arg);80 return (ret == PFM_SUCCESS);81}82 83PerfCounters PerfCounters::Create(84 const std::vector<std::string>& counter_names) {85 if (!counter_names.empty()) {86 Initialize();87 }88 89 // Valid counters will populate these arrays but we start empty90 std::vector<std::string> valid_names;91 std::vector<int> counter_ids;92 std::vector<int> leader_ids;93 94 // Resize to the maximum possible95 valid_names.reserve(counter_names.size());96 counter_ids.reserve(counter_names.size());97 98 const int kCounterMode = PFM_PLM3; // user mode only99 100 // Group leads will be assigned on demand. The idea is that once we cannot101 // create a counter descriptor, the reason is that this group has maxed out102 // so we set the group_id again to -1 and retry - giving the algorithm a103 // chance to create a new group leader to hold the next set of counters.104 int group_id = -1;105 106 // Loop through all performance counters107 for (size_t i = 0; i < counter_names.size(); ++i) {108 // we are about to push into the valid names vector109 // check if we did not reach the maximum110 if (valid_names.size() == PerfCounterValues::kMaxCounters) {111 // Log a message if we maxed out and stop adding112 GetErrorLogInstance()113 << counter_names.size() << " counters were requested. The maximum is "114 << PerfCounterValues::kMaxCounters << " and " << valid_names.size()115 << " were already added. All remaining counters will be ignored\n";116 // stop the loop and return what we have already117 break;118 }119 120 // Check if this name is empty121 const auto& name = counter_names[i];122 if (name.empty()) {123 GetErrorLogInstance()124 << "A performance counter name was the empty string\n";125 continue;126 }127 128 // Here first means first in group, ie the group leader129 const bool is_first = (group_id < 0);130 131 // This struct will be populated by libpfm from the counter string132 // and then fed into the syscall perf_event_open133 struct perf_event_attr attr {};134 attr.size = sizeof(attr);135 136 // This is the input struct to libpfm.137 pfm_perf_encode_arg_t arg{};138 arg.attr = &attr;139 const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode,140 PFM_OS_PERF_EVENT, &arg);141 if (pfm_get != PFM_SUCCESS) {142 GetErrorLogInstance()143 << "Unknown performance counter name: " << name << "\n";144 continue;145 }146 147 // We then proceed to populate the remaining fields in our attribute struct148 // Note: the man page for perf_event_create suggests inherit = true and149 // read_format = PERF_FORMAT_GROUP don't work together, but that's not the150 // case.151 attr.disabled = is_first;152 attr.inherit = true;153 attr.pinned = is_first;154 attr.exclude_kernel = true;155 attr.exclude_user = false;156 attr.exclude_hv = true;157 158 // Read all counters in a group in one read.159 attr.read_format = PERF_FORMAT_GROUP;160 161 int id = -1;162 while (id < 0) {163 static constexpr size_t kNrOfSyscallRetries = 5;164 // Retry syscall as it was interrupted often (b/64774091).165 for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries;166 ++num_retries) {167 id = perf_event_open(&attr, 0, -1, group_id, 0);168 if (id >= 0 || errno != EINTR) {169 break;170 }171 }172 if (id < 0) {173 // If the file descriptor is negative we might have reached a limit174 // in the current group. Set the group_id to -1 and retry175 if (group_id >= 0) {176 // Create a new group177 group_id = -1;178 } else {179 // At this point we have already retried to set a new group id and180 // failed. We then give up.181 break;182 }183 }184 }185 186 // We failed to get a new file descriptor. We might have reached a hard187 // hardware limit that cannot be resolved even with group multiplexing188 if (id < 0) {189 GetErrorLogInstance() << "***WARNING** Failed to get a file descriptor "190 "for performance counter "191 << name << ". Ignoring\n";192 193 // We give up on this counter but try to keep going194 // as the others would be fine195 continue;196 }197 if (group_id < 0) {198 // This is a leader, store and assign it to the current file descriptor199 leader_ids.push_back(id);200 group_id = id;201 }202 // This is a valid counter, add it to our descriptor's list203 counter_ids.push_back(id);204 valid_names.push_back(name);205 }206 207 // Loop through all group leaders activating them208 // There is another option of starting ALL counters in a process but209 // that would be far reaching an intrusion. If the user is using PMCs210 // by themselves then this would have a side effect on them. It is211 // friendlier to loop through all groups individually.212 for (int lead : leader_ids) {213 if (ioctl(lead, PERF_EVENT_IOC_ENABLE) != 0) {214 // This should never happen but if it does, we give up on the215 // entire batch as recovery would be a mess.216 GetErrorLogInstance() << "***WARNING*** Failed to start counters. "217 "Claring out all counters.\n";218 219 // Close all peformance counters220 for (int id : counter_ids) {221 ::close(id);222 }223 224 // Return an empty object so our internal state is still good and225 // the process can continue normally without impact226 return NoCounters();227 }228 }229 230 return PerfCounters(std::move(valid_names), std::move(counter_ids),231 std::move(leader_ids));232}233 234void PerfCounters::CloseCounters() const {235 if (counter_ids_.empty()) {236 return;237 }238 for (int lead : leader_ids_) {239 ioctl(lead, PERF_EVENT_IOC_DISABLE);240 }241 for (int fd : counter_ids_) {242 close(fd);243 }244}245#else // defined HAVE_LIBPFM246size_t PerfCounterValues::Read(const std::vector<int>&) { return 0; }247 248const bool PerfCounters::kSupported = false;249 250bool PerfCounters::Initialize() { return false; }251 252bool PerfCounters::IsCounterSupported(const std::string&) { return false; }253 254PerfCounters PerfCounters::Create(255 const std::vector<std::string>& counter_names) {256 if (!counter_names.empty()) {257 GetErrorLogInstance() << "Performance counters not supported.\n";258 }259 return NoCounters();260}261 262void PerfCounters::CloseCounters() const {}263#endif // defined HAVE_LIBPFM264 265PerfCountersMeasurement::PerfCountersMeasurement(266 const std::vector<std::string>& counter_names)267 : start_values_(counter_names.size()), end_values_(counter_names.size()) {268 counters_ = PerfCounters::Create(counter_names);269}270 271PerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept {272 if (this != &other) {273 CloseCounters();274 275 counter_ids_ = std::move(other.counter_ids_);276 leader_ids_ = std::move(other.leader_ids_);277 counter_names_ = std::move(other.counter_names_);278 }279 return *this;280}281} // namespace internal282} // namespace benchmark283