540 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <sys/mman.h>3#include <inttypes.h>4#include <asm/bug.h>5#include <errno.h>6#include <string.h>7#include <linux/ring_buffer.h>8#include <linux/perf_event.h>9#include <perf/mmap.h>10#include <perf/event.h>11#include <perf/evsel.h>12#include <internal/mmap.h>13#include <internal/lib.h>14#include <linux/kernel.h>15#include <linux/math64.h>16#include <linux/stringify.h>17#include "internal.h"18 19void perf_mmap__init(struct perf_mmap *map, struct perf_mmap *prev,20 bool overwrite, libperf_unmap_cb_t unmap_cb)21{22 /* Assume fields were zero initialized. */23 map->fd = -1;24 map->overwrite = overwrite;25 map->unmap_cb = unmap_cb;26 refcount_set(&map->refcnt, 0);27 if (prev)28 prev->next = map;29}30 31size_t perf_mmap__mmap_len(struct perf_mmap *map)32{33 return map->mask + 1 + page_size;34}35 36int perf_mmap__mmap(struct perf_mmap *map, struct perf_mmap_param *mp,37 int fd, struct perf_cpu cpu)38{39 map->prev = 0;40 map->mask = mp->mask;41 map->base = mmap(NULL, perf_mmap__mmap_len(map), mp->prot,42 MAP_SHARED, fd, 0);43 if (map->base == MAP_FAILED) {44 map->base = NULL;45 return -1;46 }47 48 map->fd = fd;49 map->cpu = cpu;50 return 0;51}52 53void perf_mmap__munmap(struct perf_mmap *map)54{55 if (!map)56 return;57 58 zfree(&map->event_copy);59 map->event_copy_sz = 0;60 if (map->base) {61 munmap(map->base, perf_mmap__mmap_len(map));62 map->base = NULL;63 map->fd = -1;64 refcount_set(&map->refcnt, 0);65 }66 if (map->unmap_cb)67 map->unmap_cb(map);68}69 70void perf_mmap__get(struct perf_mmap *map)71{72 refcount_inc(&map->refcnt);73}74 75void perf_mmap__put(struct perf_mmap *map)76{77 BUG_ON(map->base && refcount_read(&map->refcnt) == 0);78 79 if (refcount_dec_and_test(&map->refcnt))80 perf_mmap__munmap(map);81}82 83static inline void perf_mmap__write_tail(struct perf_mmap *md, u64 tail)84{85 ring_buffer_write_tail(md->base, tail);86}87 88u64 perf_mmap__read_head(struct perf_mmap *map)89{90 return ring_buffer_read_head(map->base);91}92 93static bool perf_mmap__empty(struct perf_mmap *map)94{95 struct perf_event_mmap_page *pc = map->base;96 97 return perf_mmap__read_head(map) == map->prev && !pc->aux_size;98}99 100void perf_mmap__consume(struct perf_mmap *map)101{102 if (!map->overwrite) {103 u64 old = map->prev;104 105 perf_mmap__write_tail(map, old);106 }107 108 if (refcount_read(&map->refcnt) == 1 && perf_mmap__empty(map))109 perf_mmap__put(map);110}111 112static int overwrite_rb_find_range(void *buf, int mask, u64 *start, u64 *end)113{114 struct perf_event_header *pheader;115 u64 evt_head = *start;116 int size = mask + 1;117 118 pr_debug2("%s: buf=%p, start=%"PRIx64"\n", __func__, buf, *start);119 pheader = (struct perf_event_header *)(buf + (*start & mask));120 while (true) {121 if (evt_head - *start >= (unsigned int)size) {122 pr_debug("Finished reading overwrite ring buffer: rewind\n");123 if (evt_head - *start > (unsigned int)size)124 evt_head -= pheader->size;125 *end = evt_head;126 return 0;127 }128 129 pheader = (struct perf_event_header *)(buf + (evt_head & mask));130 131 if (pheader->size == 0) {132 pr_debug("Finished reading overwrite ring buffer: get start\n");133 *end = evt_head;134 return 0;135 }136 137 evt_head += pheader->size;138 pr_debug3("move evt_head: %"PRIx64"\n", evt_head);139 }140 WARN_ONCE(1, "Shouldn't get here\n");141 return -1;142}143 144/*145 * Report the start and end of the available data in ringbuffer146 */147static int __perf_mmap__read_init(struct perf_mmap *md)148{149 u64 head = perf_mmap__read_head(md);150 u64 old = md->prev;151 unsigned char *data = md->base + page_size;152 unsigned long size;153 154 md->start = md->overwrite ? head : old;155 md->end = md->overwrite ? old : head;156 157 if ((md->end - md->start) < md->flush)158 return -EAGAIN;159 160 size = md->end - md->start;161 if (size > (unsigned long)(md->mask) + 1) {162 if (!md->overwrite) {163 WARN_ONCE(1, "failed to keep up with mmap data. (warn only once)\n");164 165 md->prev = head;166 perf_mmap__consume(md);167 return -EAGAIN;168 }169 170 /*171 * Backward ring buffer is full. We still have a chance to read172 * most of data from it.173 */174 if (overwrite_rb_find_range(data, md->mask, &md->start, &md->end))175 return -EINVAL;176 }177 178 return 0;179}180 181int perf_mmap__read_init(struct perf_mmap *map)182{183 /*184 * Check if event was unmapped due to a POLLHUP/POLLERR.185 */186 if (!refcount_read(&map->refcnt))187 return -ENOENT;188 189 return __perf_mmap__read_init(map);190}191 192/*193 * Mandatory for overwrite mode194 * The direction of overwrite mode is backward.195 * The last perf_mmap__read() will set tail to map->core.prev.196 * Need to correct the map->core.prev to head which is the end of next read.197 */198void perf_mmap__read_done(struct perf_mmap *map)199{200 /*201 * Check if event was unmapped due to a POLLHUP/POLLERR.202 */203 if (!refcount_read(&map->refcnt))204 return;205 206 map->prev = perf_mmap__read_head(map);207}208 209/* When check_messup is true, 'end' must points to a good entry */210static union perf_event *perf_mmap__read(struct perf_mmap *map,211 u64 *startp, u64 end)212{213 unsigned char *data = map->base + page_size;214 union perf_event *event = NULL;215 int diff = end - *startp;216 217 if (diff >= (int)sizeof(event->header)) {218 size_t size;219 220 event = (union perf_event *)&data[*startp & map->mask];221 size = event->header.size;222 223 if (size < sizeof(event->header) || diff < (int)size)224 return NULL;225 226 /*227 * Event straddles the mmap boundary -- header should always228 * be inside due to u64 alignment of output.229 */230 if ((*startp & map->mask) + size != ((*startp + size) & map->mask)) {231 unsigned int offset = *startp;232 unsigned int len = size, cpy;233 void *dst = map->event_copy;234 235 if (size > map->event_copy_sz) {236 dst = realloc(map->event_copy, size);237 if (!dst)238 return NULL;239 map->event_copy = dst;240 map->event_copy_sz = size;241 }242 243 do {244 cpy = min(map->mask + 1 - (offset & map->mask), len);245 memcpy(dst, &data[offset & map->mask], cpy);246 offset += cpy;247 dst += cpy;248 len -= cpy;249 } while (len);250 251 event = (union perf_event *)map->event_copy;252 }253 254 *startp += size;255 }256 257 return event;258}259 260/*261 * Read event from ring buffer one by one.262 * Return one event for each call.263 *264 * Usage:265 * perf_mmap__read_init()266 * while(event = perf_mmap__read_event()) {267 * //process the event268 * perf_mmap__consume()269 * }270 * perf_mmap__read_done()271 */272union perf_event *perf_mmap__read_event(struct perf_mmap *map)273{274 union perf_event *event;275 276 /*277 * Check if event was unmapped due to a POLLHUP/POLLERR.278 */279 if (!refcount_read(&map->refcnt))280 return NULL;281 282 /* non-overwrite doesn't pause the ringbuffer */283 if (!map->overwrite)284 map->end = perf_mmap__read_head(map);285 286 event = perf_mmap__read(map, &map->start, map->end);287 288 if (!map->overwrite)289 map->prev = map->start;290 291 return event;292}293 294#if defined(__i386__) || defined(__x86_64__)295static u64 read_perf_counter(unsigned int counter)296{297 unsigned int low, high;298 299 asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));300 301 return low | ((u64)high) << 32;302}303 304static u64 read_timestamp(void)305{306 unsigned int low, high;307 308 asm volatile("rdtsc" : "=a" (low), "=d" (high));309 310 return low | ((u64)high) << 32;311}312#elif defined(__aarch64__)313#define read_sysreg(r) ({ \314 u64 __val; \315 asm volatile("mrs %0, " __stringify(r) : "=r" (__val)); \316 __val; \317})318 319static u64 read_pmccntr(void)320{321 return read_sysreg(pmccntr_el0);322}323 324#define PMEVCNTR_READ(idx) \325 static u64 read_pmevcntr_##idx(void) { \326 return read_sysreg(pmevcntr##idx##_el0); \327 }328 329PMEVCNTR_READ(0);330PMEVCNTR_READ(1);331PMEVCNTR_READ(2);332PMEVCNTR_READ(3);333PMEVCNTR_READ(4);334PMEVCNTR_READ(5);335PMEVCNTR_READ(6);336PMEVCNTR_READ(7);337PMEVCNTR_READ(8);338PMEVCNTR_READ(9);339PMEVCNTR_READ(10);340PMEVCNTR_READ(11);341PMEVCNTR_READ(12);342PMEVCNTR_READ(13);343PMEVCNTR_READ(14);344PMEVCNTR_READ(15);345PMEVCNTR_READ(16);346PMEVCNTR_READ(17);347PMEVCNTR_READ(18);348PMEVCNTR_READ(19);349PMEVCNTR_READ(20);350PMEVCNTR_READ(21);351PMEVCNTR_READ(22);352PMEVCNTR_READ(23);353PMEVCNTR_READ(24);354PMEVCNTR_READ(25);355PMEVCNTR_READ(26);356PMEVCNTR_READ(27);357PMEVCNTR_READ(28);358PMEVCNTR_READ(29);359PMEVCNTR_READ(30);360 361/*362 * Read a value direct from PMEVCNTR<idx>363 */364static u64 read_perf_counter(unsigned int counter)365{366 static u64 (* const read_f[])(void) = {367 read_pmevcntr_0,368 read_pmevcntr_1,369 read_pmevcntr_2,370 read_pmevcntr_3,371 read_pmevcntr_4,372 read_pmevcntr_5,373 read_pmevcntr_6,374 read_pmevcntr_7,375 read_pmevcntr_8,376 read_pmevcntr_9,377 read_pmevcntr_10,378 read_pmevcntr_11,379 read_pmevcntr_13,380 read_pmevcntr_12,381 read_pmevcntr_14,382 read_pmevcntr_15,383 read_pmevcntr_16,384 read_pmevcntr_17,385 read_pmevcntr_18,386 read_pmevcntr_19,387 read_pmevcntr_20,388 read_pmevcntr_21,389 read_pmevcntr_22,390 read_pmevcntr_23,391 read_pmevcntr_24,392 read_pmevcntr_25,393 read_pmevcntr_26,394 read_pmevcntr_27,395 read_pmevcntr_28,396 read_pmevcntr_29,397 read_pmevcntr_30,398 read_pmccntr399 };400 401 if (counter < ARRAY_SIZE(read_f))402 return (read_f[counter])();403 404 return 0;405}406 407static u64 read_timestamp(void) { return read_sysreg(cntvct_el0); }408 409/* __riscv_xlen contains the witdh of the native base integer, here 64-bit */410#elif defined(__riscv) && __riscv_xlen == 64411 412/* TODO: implement rv32 support */413 414#define CSR_CYCLE 0xc00415#define CSR_TIME 0xc01416 417#define csr_read(csr) \418({ \419 register unsigned long __v; \420 __asm__ __volatile__ ("csrr %0, %1" \421 : "=r" (__v) \422 : "i" (csr) : ); \423 __v; \424})425 426static unsigned long csr_read_num(int csr_num)427{428#define switchcase_csr_read(__csr_num, __val) {\429 case __csr_num: \430 __val = csr_read(__csr_num); \431 break; }432#define switchcase_csr_read_2(__csr_num, __val) {\433 switchcase_csr_read(__csr_num + 0, __val) \434 switchcase_csr_read(__csr_num + 1, __val)}435#define switchcase_csr_read_4(__csr_num, __val) {\436 switchcase_csr_read_2(__csr_num + 0, __val) \437 switchcase_csr_read_2(__csr_num + 2, __val)}438#define switchcase_csr_read_8(__csr_num, __val) {\439 switchcase_csr_read_4(__csr_num + 0, __val) \440 switchcase_csr_read_4(__csr_num + 4, __val)}441#define switchcase_csr_read_16(__csr_num, __val) {\442 switchcase_csr_read_8(__csr_num + 0, __val) \443 switchcase_csr_read_8(__csr_num + 8, __val)}444#define switchcase_csr_read_32(__csr_num, __val) {\445 switchcase_csr_read_16(__csr_num + 0, __val) \446 switchcase_csr_read_16(__csr_num + 16, __val)}447 448 unsigned long ret = 0;449 450 switch (csr_num) {451 switchcase_csr_read_32(CSR_CYCLE, ret)452 default:453 break;454 }455 456 return ret;457#undef switchcase_csr_read_32458#undef switchcase_csr_read_16459#undef switchcase_csr_read_8460#undef switchcase_csr_read_4461#undef switchcase_csr_read_2462#undef switchcase_csr_read463}464 465static u64 read_perf_counter(unsigned int counter)466{467 return csr_read_num(CSR_CYCLE + counter);468}469 470static u64 read_timestamp(void)471{472 return csr_read_num(CSR_TIME);473}474 475#else476static u64 read_perf_counter(unsigned int counter __maybe_unused) { return 0; }477static u64 read_timestamp(void) { return 0; }478#endif479 480int perf_mmap__read_self(struct perf_mmap *map, struct perf_counts_values *count)481{482 struct perf_event_mmap_page *pc = map->base;483 u32 seq, idx, time_mult = 0, time_shift = 0;484 u64 cnt, cyc = 0, time_offset = 0, time_cycles = 0, time_mask = ~0ULL;485 486 if (!pc || !pc->cap_user_rdpmc)487 return -1;488 489 do {490 seq = READ_ONCE(pc->lock);491 barrier();492 493 count->ena = READ_ONCE(pc->time_enabled);494 count->run = READ_ONCE(pc->time_running);495 496 if (pc->cap_user_time && count->ena != count->run) {497 cyc = read_timestamp();498 time_mult = READ_ONCE(pc->time_mult);499 time_shift = READ_ONCE(pc->time_shift);500 time_offset = READ_ONCE(pc->time_offset);501 502 if (pc->cap_user_time_short) {503 time_cycles = READ_ONCE(pc->time_cycles);504 time_mask = READ_ONCE(pc->time_mask);505 }506 }507 508 idx = READ_ONCE(pc->index);509 cnt = READ_ONCE(pc->offset);510 if (pc->cap_user_rdpmc && idx) {511 s64 evcnt = read_perf_counter(idx - 1);512 u16 width = READ_ONCE(pc->pmc_width);513 514 evcnt <<= 64 - width;515 evcnt >>= 64 - width;516 cnt += evcnt;517 } else518 return -1;519 520 barrier();521 } while (READ_ONCE(pc->lock) != seq);522 523 if (count->ena != count->run) {524 u64 delta;525 526 /* Adjust for cap_usr_time_short, a nop if not */527 cyc = time_cycles + ((cyc - time_cycles) & time_mask);528 529 delta = time_offset + mul_u64_u32_shr(cyc, time_mult, time_shift);530 531 count->ena += delta;532 if (idx)533 count->run += delta;534 }535 536 count->val = cnt;537 538 return 0;539}540