558 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <errno.h>3#include <unistd.h>4#include <sys/syscall.h>5#include <perf/evsel.h>6#include <perf/cpumap.h>7#include <perf/threadmap.h>8#include <linux/list.h>9#include <internal/evsel.h>10#include <linux/zalloc.h>11#include <stdlib.h>12#include <internal/xyarray.h>13#include <internal/cpumap.h>14#include <internal/mmap.h>15#include <internal/threadmap.h>16#include <internal/lib.h>17#include <linux/string.h>18#include <sys/ioctl.h>19#include <sys/mman.h>20#include <asm/bug.h>21 22void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr,23 int idx)24{25 INIT_LIST_HEAD(&evsel->node);26 evsel->attr = *attr;27 evsel->idx = idx;28 evsel->leader = evsel;29}30 31struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr)32{33 struct perf_evsel *evsel = zalloc(sizeof(*evsel));34 35 if (evsel != NULL)36 perf_evsel__init(evsel, attr, 0);37 38 return evsel;39}40 41void perf_evsel__delete(struct perf_evsel *evsel)42{43 free(evsel);44}45 46#define FD(_evsel, _cpu_map_idx, _thread) \47 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread))48#define MMAP(_evsel, _cpu_map_idx, _thread) \49 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \50 : NULL)51 52int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)53{54 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));55 56 if (evsel->fd) {57 int idx, thread;58 59 for (idx = 0; idx < ncpus; idx++) {60 for (thread = 0; thread < nthreads; thread++) {61 int *fd = FD(evsel, idx, thread);62 63 if (fd)64 *fd = -1;65 }66 }67 }68 69 return evsel->fd != NULL ? 0 : -ENOMEM;70}71 72static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads)73{74 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap));75 76 return evsel->mmap != NULL ? 0 : -ENOMEM;77}78 79static int80sys_perf_event_open(struct perf_event_attr *attr,81 pid_t pid, struct perf_cpu cpu, int group_fd,82 unsigned long flags)83{84 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags);85}86 87static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd)88{89 struct perf_evsel *leader = evsel->leader;90 int *fd;91 92 if (evsel == leader) {93 *group_fd = -1;94 return 0;95 }96 97 /*98 * Leader must be already processed/open,99 * if not it's a bug.100 */101 if (!leader->fd)102 return -ENOTCONN;103 104 fd = FD(leader, cpu_map_idx, thread);105 if (fd == NULL || *fd == -1)106 return -EBADF;107 108 *group_fd = *fd;109 110 return 0;111}112 113int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus,114 struct perf_thread_map *threads)115{116 struct perf_cpu cpu;117 int idx, thread, err = 0;118 119 if (cpus == NULL) {120 static struct perf_cpu_map *empty_cpu_map;121 122 if (empty_cpu_map == NULL) {123 empty_cpu_map = perf_cpu_map__new_any_cpu();124 if (empty_cpu_map == NULL)125 return -ENOMEM;126 }127 128 cpus = empty_cpu_map;129 }130 131 if (threads == NULL) {132 static struct perf_thread_map *empty_thread_map;133 134 if (empty_thread_map == NULL) {135 empty_thread_map = perf_thread_map__new_dummy();136 if (empty_thread_map == NULL)137 return -ENOMEM;138 }139 140 threads = empty_thread_map;141 }142 143 if (evsel->fd == NULL &&144 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)145 return -ENOMEM;146 147 perf_cpu_map__for_each_cpu(cpu, idx, cpus) {148 for (thread = 0; thread < threads->nr; thread++) {149 int fd, group_fd, *evsel_fd;150 151 evsel_fd = FD(evsel, idx, thread);152 if (evsel_fd == NULL) {153 err = -EINVAL;154 goto out;155 }156 157 err = get_group_fd(evsel, idx, thread, &group_fd);158 if (err < 0)159 goto out;160 161 fd = sys_perf_event_open(&evsel->attr,162 threads->map[thread].pid,163 cpu, group_fd, 0);164 165 if (fd < 0) {166 err = -errno;167 goto out;168 }169 170 *evsel_fd = fd;171 }172 }173out:174 if (err)175 perf_evsel__close(evsel);176 177 return err;178}179 180static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx)181{182 int thread;183 184 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) {185 int *fd = FD(evsel, cpu_map_idx, thread);186 187 if (fd && *fd >= 0) {188 close(*fd);189 *fd = -1;190 }191 }192}193 194void perf_evsel__close_fd(struct perf_evsel *evsel)195{196 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++)197 perf_evsel__close_fd_cpu(evsel, idx);198}199 200void perf_evsel__free_fd(struct perf_evsel *evsel)201{202 xyarray__delete(evsel->fd);203 evsel->fd = NULL;204}205 206void perf_evsel__close(struct perf_evsel *evsel)207{208 if (evsel->fd == NULL)209 return;210 211 perf_evsel__close_fd(evsel);212 perf_evsel__free_fd(evsel);213}214 215void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx)216{217 if (evsel->fd == NULL)218 return;219 220 perf_evsel__close_fd_cpu(evsel, cpu_map_idx);221}222 223void perf_evsel__munmap(struct perf_evsel *evsel)224{225 int idx, thread;226 227 if (evsel->fd == NULL || evsel->mmap == NULL)228 return;229 230 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {231 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {232 int *fd = FD(evsel, idx, thread);233 234 if (fd == NULL || *fd < 0)235 continue;236 237 perf_mmap__munmap(MMAP(evsel, idx, thread));238 }239 }240 241 xyarray__delete(evsel->mmap);242 evsel->mmap = NULL;243}244 245int perf_evsel__mmap(struct perf_evsel *evsel, int pages)246{247 int ret, idx, thread;248 struct perf_mmap_param mp = {249 .prot = PROT_READ | PROT_WRITE,250 .mask = (pages * page_size) - 1,251 };252 253 if (evsel->fd == NULL || evsel->mmap)254 return -EINVAL;255 256 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0)257 return -ENOMEM;258 259 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {260 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {261 int *fd = FD(evsel, idx, thread);262 struct perf_mmap *map;263 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx);264 265 if (fd == NULL || *fd < 0)266 continue;267 268 map = MMAP(evsel, idx, thread);269 perf_mmap__init(map, NULL, false, NULL);270 271 ret = perf_mmap__mmap(map, &mp, *fd, cpu);272 if (ret) {273 perf_evsel__munmap(evsel);274 return ret;275 }276 }277 }278 279 return 0;280}281 282void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread)283{284 int *fd = FD(evsel, cpu_map_idx, thread);285 286 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL)287 return NULL;288 289 return MMAP(evsel, cpu_map_idx, thread)->base;290}291 292int perf_evsel__read_size(struct perf_evsel *evsel)293{294 u64 read_format = evsel->attr.read_format;295 int entry = sizeof(u64); /* value */296 int size = 0;297 int nr = 1;298 299 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)300 size += sizeof(u64);301 302 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)303 size += sizeof(u64);304 305 if (read_format & PERF_FORMAT_ID)306 entry += sizeof(u64);307 308 if (read_format & PERF_FORMAT_LOST)309 entry += sizeof(u64);310 311 if (read_format & PERF_FORMAT_GROUP) {312 nr = evsel->nr_members;313 size += sizeof(u64);314 }315 316 size += entry * nr;317 return size;318}319 320/* This only reads values for the leader */321static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,322 int thread, struct perf_counts_values *count)323{324 size_t size = perf_evsel__read_size(evsel);325 int *fd = FD(evsel, cpu_map_idx, thread);326 u64 read_format = evsel->attr.read_format;327 u64 *data;328 int idx = 1;329 330 if (fd == NULL || *fd < 0)331 return -EINVAL;332 333 data = calloc(1, size);334 if (data == NULL)335 return -ENOMEM;336 337 if (readn(*fd, data, size) <= 0) {338 free(data);339 return -errno;340 }341 342 /*343 * This reads only the leader event intentionally since we don't have344 * perf counts values for sibling events.345 */346 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)347 count->ena = data[idx++];348 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)349 count->run = data[idx++];350 351 /* value is always available */352 count->val = data[idx++];353 if (read_format & PERF_FORMAT_ID)354 count->id = data[idx++];355 if (read_format & PERF_FORMAT_LOST)356 count->lost = data[idx++];357 358 free(data);359 return 0;360}361 362/*363 * The perf read format is very flexible. It needs to set the proper364 * values according to the read format.365 */366static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf,367 struct perf_counts_values *count)368{369 u64 read_format = evsel->attr.read_format;370 int n = 0;371 372 count->val = buf[n++];373 374 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)375 count->ena = buf[n++];376 377 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)378 count->run = buf[n++];379 380 if (read_format & PERF_FORMAT_ID)381 count->id = buf[n++];382 383 if (read_format & PERF_FORMAT_LOST)384 count->lost = buf[n++];385}386 387int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,388 struct perf_counts_values *count)389{390 size_t size = perf_evsel__read_size(evsel);391 int *fd = FD(evsel, cpu_map_idx, thread);392 u64 read_format = evsel->attr.read_format;393 struct perf_counts_values buf;394 395 memset(count, 0, sizeof(*count));396 397 if (fd == NULL || *fd < 0)398 return -EINVAL;399 400 if (read_format & PERF_FORMAT_GROUP)401 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);402 403 if (MMAP(evsel, cpu_map_idx, thread) &&404 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&405 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))406 return 0;407 408 if (readn(*fd, buf.values, size) <= 0)409 return -errno;410 411 perf_evsel__adjust_values(evsel, buf.values, count);412 return 0;413}414 415static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg,416 int cpu_map_idx, int thread)417{418 int *fd = FD(evsel, cpu_map_idx, thread);419 420 if (fd == NULL || *fd < 0)421 return -1;422 423 return ioctl(*fd, ioc, arg);424}425 426static int perf_evsel__run_ioctl(struct perf_evsel *evsel,427 int ioc, void *arg,428 int cpu_map_idx)429{430 int thread;431 432 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {433 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread);434 435 if (err)436 return err;437 }438 439 return 0;440}441 442int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx)443{444 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx);445}446 447int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread)448{449 struct perf_cpu cpu __maybe_unused;450 int idx;451 int err;452 453 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) {454 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread);455 if (err)456 return err;457 }458 459 return 0;460}461 462int perf_evsel__enable(struct perf_evsel *evsel)463{464 int i;465 int err = 0;466 467 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)468 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i);469 return err;470}471 472int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx)473{474 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx);475}476 477int perf_evsel__disable(struct perf_evsel *evsel)478{479 int i;480 int err = 0;481 482 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)483 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i);484 return err;485}486 487int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter)488{489 int err = 0, i;490 491 for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++)492 err = perf_evsel__run_ioctl(evsel,493 PERF_EVENT_IOC_SET_FILTER,494 (void *)filter, i);495 return err;496}497 498struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel)499{500 return evsel->cpus;501}502 503struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel)504{505 return evsel->threads;506}507 508struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel)509{510 return &evsel->attr;511}512 513int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)514{515 if (ncpus == 0 || nthreads == 0)516 return 0;517 518 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));519 if (evsel->sample_id == NULL)520 return -ENOMEM;521 522 evsel->id = zalloc(ncpus * nthreads * sizeof(u64));523 if (evsel->id == NULL) {524 xyarray__delete(evsel->sample_id);525 evsel->sample_id = NULL;526 return -ENOMEM;527 }528 529 return 0;530}531 532void perf_evsel__free_id(struct perf_evsel *evsel)533{534 xyarray__delete(evsel->sample_id);535 evsel->sample_id = NULL;536 zfree(&evsel->id);537 evsel->ids = 0;538}539 540void perf_counts_values__scale(struct perf_counts_values *count,541 bool scale, __s8 *pscaled)542{543 s8 scaled = 0;544 545 if (scale) {546 if (count->run == 0) {547 scaled = -1;548 count->val = 0;549 } else if (count->run < count->ena) {550 scaled = 1;551 count->val = (u64)((double)count->val * count->ena / count->run);552 }553 }554 555 if (pscaled)556 *pscaled = scaled;557}558