2882 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * auxtrace.c: AUX area trace support4 * Copyright (c) 2013-2015, Intel Corporation.5 */6 7#include <inttypes.h>8#include <sys/types.h>9#include <sys/mman.h>10#include <stdbool.h>11#include <string.h>12#include <limits.h>13#include <errno.h>14 15#include <linux/kernel.h>16#include <linux/perf_event.h>17#include <linux/types.h>18#include <linux/bitops.h>19#include <linux/log2.h>20#include <linux/string.h>21#include <linux/time64.h>22 23#include <sys/param.h>24#include <stdlib.h>25#include <stdio.h>26#include <linux/list.h>27#include <linux/zalloc.h>28 29#include "config.h"30#include "evlist.h"31#include "dso.h"32#include "map.h"33#include "pmu.h"34#include "evsel.h"35#include "evsel_config.h"36#include "symbol.h"37#include "util/perf_api_probe.h"38#include "util/synthetic-events.h"39#include "thread_map.h"40#include "asm/bug.h"41#include "auxtrace.h"42 43#include <linux/hash.h>44 45#include "event.h"46#include "record.h"47#include "session.h"48#include "debug.h"49#include <subcmd/parse-options.h>50 51#include "cs-etm.h"52#include "intel-pt.h"53#include "intel-bts.h"54#include "arm-spe.h"55#include "hisi-ptt.h"56#include "s390-cpumsf.h"57#include "util/mmap.h"58 59#include <linux/ctype.h>60#include "symbol/kallsyms.h"61#include <internal/lib.h>62#include "util/sample.h"63 64/*65 * Make a group from 'leader' to 'last', requiring that the events were not66 * already grouped to a different leader.67 */68static int evlist__regroup(struct evlist *evlist, struct evsel *leader, struct evsel *last)69{70 struct evsel *evsel;71 bool grp;72 73 if (!evsel__is_group_leader(leader))74 return -EINVAL;75 76 grp = false;77 evlist__for_each_entry(evlist, evsel) {78 if (grp) {79 if (!(evsel__leader(evsel) == leader ||80 (evsel__leader(evsel) == evsel &&81 evsel->core.nr_members <= 1)))82 return -EINVAL;83 } else if (evsel == leader) {84 grp = true;85 }86 if (evsel == last)87 break;88 }89 90 grp = false;91 evlist__for_each_entry(evlist, evsel) {92 if (grp) {93 if (!evsel__has_leader(evsel, leader)) {94 evsel__set_leader(evsel, leader);95 if (leader->core.nr_members < 1)96 leader->core.nr_members = 1;97 leader->core.nr_members += 1;98 }99 } else if (evsel == leader) {100 grp = true;101 }102 if (evsel == last)103 break;104 }105 106 return 0;107}108 109static bool auxtrace__dont_decode(struct perf_session *session)110{111 return !session->itrace_synth_opts ||112 session->itrace_synth_opts->dont_decode;113}114 115int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,116 struct auxtrace_mmap_params *mp,117 void *userpg, int fd)118{119 struct perf_event_mmap_page *pc = userpg;120 121 WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n");122 123 mm->userpg = userpg;124 mm->mask = mp->mask;125 mm->len = mp->len;126 mm->prev = 0;127 mm->idx = mp->idx;128 mm->tid = mp->tid;129 mm->cpu = mp->cpu.cpu;130 131 if (!mp->len || !mp->mmap_needed) {132 mm->base = NULL;133 return 0;134 }135 136 pc->aux_offset = mp->offset;137 pc->aux_size = mp->len;138 139 mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset);140 if (mm->base == MAP_FAILED) {141 pr_debug2("failed to mmap AUX area\n");142 mm->base = NULL;143 return -1;144 }145 146 return 0;147}148 149void auxtrace_mmap__munmap(struct auxtrace_mmap *mm)150{151 if (mm->base) {152 munmap(mm->base, mm->len);153 mm->base = NULL;154 }155}156 157void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,158 off_t auxtrace_offset,159 unsigned int auxtrace_pages,160 bool auxtrace_overwrite)161{162 if (auxtrace_pages) {163 mp->offset = auxtrace_offset;164 mp->len = auxtrace_pages * (size_t)page_size;165 mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0;166 mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE);167 pr_debug2("AUX area mmap length %zu\n", mp->len);168 } else {169 mp->len = 0;170 }171}172 173void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,174 struct evlist *evlist,175 struct evsel *evsel, int idx)176{177 bool per_cpu = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus);178 179 mp->mmap_needed = evsel->needs_auxtrace_mmap;180 181 if (!mp->mmap_needed)182 return;183 184 mp->idx = idx;185 186 if (per_cpu) {187 mp->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx);188 if (evlist->core.threads)189 mp->tid = perf_thread_map__pid(evlist->core.threads, 0);190 else191 mp->tid = -1;192 } else {193 mp->cpu.cpu = -1;194 mp->tid = perf_thread_map__pid(evlist->core.threads, idx);195 }196}197 198#define AUXTRACE_INIT_NR_QUEUES 32199 200static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues)201{202 struct auxtrace_queue *queue_array;203 unsigned int max_nr_queues, i;204 205 max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue);206 if (nr_queues > max_nr_queues)207 return NULL;208 209 queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue));210 if (!queue_array)211 return NULL;212 213 for (i = 0; i < nr_queues; i++) {214 INIT_LIST_HEAD(&queue_array[i].head);215 queue_array[i].priv = NULL;216 }217 218 return queue_array;219}220 221int auxtrace_queues__init_nr(struct auxtrace_queues *queues, int nr_queues)222{223 queues->nr_queues = nr_queues;224 queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues);225 if (!queues->queue_array)226 return -ENOMEM;227 return 0;228}229 230int auxtrace_queues__init(struct auxtrace_queues *queues)231{232 return auxtrace_queues__init_nr(queues, AUXTRACE_INIT_NR_QUEUES);233}234 235static int auxtrace_queues__grow(struct auxtrace_queues *queues,236 unsigned int new_nr_queues)237{238 unsigned int nr_queues = queues->nr_queues;239 struct auxtrace_queue *queue_array;240 unsigned int i;241 242 if (!nr_queues)243 nr_queues = AUXTRACE_INIT_NR_QUEUES;244 245 while (nr_queues && nr_queues < new_nr_queues)246 nr_queues <<= 1;247 248 if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues)249 return -EINVAL;250 251 queue_array = auxtrace_alloc_queue_array(nr_queues);252 if (!queue_array)253 return -ENOMEM;254 255 for (i = 0; i < queues->nr_queues; i++) {256 list_splice_tail(&queues->queue_array[i].head,257 &queue_array[i].head);258 queue_array[i].tid = queues->queue_array[i].tid;259 queue_array[i].cpu = queues->queue_array[i].cpu;260 queue_array[i].set = queues->queue_array[i].set;261 queue_array[i].priv = queues->queue_array[i].priv;262 }263 264 queues->nr_queues = nr_queues;265 queues->queue_array = queue_array;266 267 return 0;268}269 270static void *auxtrace_copy_data(u64 size, struct perf_session *session)271{272 int fd = perf_data__fd(session->data);273 void *p;274 ssize_t ret;275 276 if (size > SSIZE_MAX)277 return NULL;278 279 p = malloc(size);280 if (!p)281 return NULL;282 283 ret = readn(fd, p, size);284 if (ret != (ssize_t)size) {285 free(p);286 return NULL;287 }288 289 return p;290}291 292static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues,293 unsigned int idx,294 struct auxtrace_buffer *buffer)295{296 struct auxtrace_queue *queue;297 int err;298 299 if (idx >= queues->nr_queues) {300 err = auxtrace_queues__grow(queues, idx + 1);301 if (err)302 return err;303 }304 305 queue = &queues->queue_array[idx];306 307 if (!queue->set) {308 queue->set = true;309 queue->tid = buffer->tid;310 queue->cpu = buffer->cpu.cpu;311 }312 313 buffer->buffer_nr = queues->next_buffer_nr++;314 315 list_add_tail(&buffer->list, &queue->head);316 317 queues->new_data = true;318 queues->populated = true;319 320 return 0;321}322 323/* Limit buffers to 32MiB on 32-bit */324#define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024)325 326static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues,327 unsigned int idx,328 struct auxtrace_buffer *buffer)329{330 u64 sz = buffer->size;331 bool consecutive = false;332 struct auxtrace_buffer *b;333 int err;334 335 while (sz > BUFFER_LIMIT_FOR_32_BIT) {336 b = memdup(buffer, sizeof(struct auxtrace_buffer));337 if (!b)338 return -ENOMEM;339 b->size = BUFFER_LIMIT_FOR_32_BIT;340 b->consecutive = consecutive;341 err = auxtrace_queues__queue_buffer(queues, idx, b);342 if (err) {343 auxtrace_buffer__free(b);344 return err;345 }346 buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT;347 sz -= BUFFER_LIMIT_FOR_32_BIT;348 consecutive = true;349 }350 351 buffer->size = sz;352 buffer->consecutive = consecutive;353 354 return 0;355}356 357static bool filter_cpu(struct perf_session *session, struct perf_cpu cpu)358{359 unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap;360 361 return cpu_bitmap && cpu.cpu != -1 && !test_bit(cpu.cpu, cpu_bitmap);362}363 364static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues,365 struct perf_session *session,366 unsigned int idx,367 struct auxtrace_buffer *buffer,368 struct auxtrace_buffer **buffer_ptr)369{370 int err = -ENOMEM;371 372 if (filter_cpu(session, buffer->cpu))373 return 0;374 375 buffer = memdup(buffer, sizeof(*buffer));376 if (!buffer)377 return -ENOMEM;378 379 if (session->one_mmap) {380 buffer->data = buffer->data_offset - session->one_mmap_offset +381 session->one_mmap_addr;382 } else if (perf_data__is_pipe(session->data)) {383 buffer->data = auxtrace_copy_data(buffer->size, session);384 if (!buffer->data)385 goto out_free;386 buffer->data_needs_freeing = true;387 } else if (BITS_PER_LONG == 32 &&388 buffer->size > BUFFER_LIMIT_FOR_32_BIT) {389 err = auxtrace_queues__split_buffer(queues, idx, buffer);390 if (err)391 goto out_free;392 }393 394 err = auxtrace_queues__queue_buffer(queues, idx, buffer);395 if (err)396 goto out_free;397 398 /* FIXME: Doesn't work for split buffer */399 if (buffer_ptr)400 *buffer_ptr = buffer;401 402 return 0;403 404out_free:405 auxtrace_buffer__free(buffer);406 return err;407}408 409int auxtrace_queues__add_event(struct auxtrace_queues *queues,410 struct perf_session *session,411 union perf_event *event, off_t data_offset,412 struct auxtrace_buffer **buffer_ptr)413{414 struct auxtrace_buffer buffer = {415 .pid = -1,416 .tid = event->auxtrace.tid,417 .cpu = { event->auxtrace.cpu },418 .data_offset = data_offset,419 .offset = event->auxtrace.offset,420 .reference = event->auxtrace.reference,421 .size = event->auxtrace.size,422 };423 unsigned int idx = event->auxtrace.idx;424 425 return auxtrace_queues__add_buffer(queues, session, idx, &buffer,426 buffer_ptr);427}428 429static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues,430 struct perf_session *session,431 off_t file_offset, size_t sz)432{433 union perf_event *event;434 int err;435 char buf[PERF_SAMPLE_MAX_SIZE];436 437 err = perf_session__peek_event(session, file_offset, buf,438 PERF_SAMPLE_MAX_SIZE, &event, NULL);439 if (err)440 return err;441 442 if (event->header.type == PERF_RECORD_AUXTRACE) {443 if (event->header.size < sizeof(struct perf_record_auxtrace) ||444 event->header.size != sz) {445 err = -EINVAL;446 goto out;447 }448 file_offset += event->header.size;449 err = auxtrace_queues__add_event(queues, session, event,450 file_offset, NULL);451 }452out:453 return err;454}455 456void auxtrace_queues__free(struct auxtrace_queues *queues)457{458 unsigned int i;459 460 for (i = 0; i < queues->nr_queues; i++) {461 while (!list_empty(&queues->queue_array[i].head)) {462 struct auxtrace_buffer *buffer;463 464 buffer = list_entry(queues->queue_array[i].head.next,465 struct auxtrace_buffer, list);466 list_del_init(&buffer->list);467 auxtrace_buffer__free(buffer);468 }469 }470 471 zfree(&queues->queue_array);472 queues->nr_queues = 0;473}474 475static void auxtrace_heapify(struct auxtrace_heap_item *heap_array,476 unsigned int pos, unsigned int queue_nr,477 u64 ordinal)478{479 unsigned int parent;480 481 while (pos) {482 parent = (pos - 1) >> 1;483 if (heap_array[parent].ordinal <= ordinal)484 break;485 heap_array[pos] = heap_array[parent];486 pos = parent;487 }488 heap_array[pos].queue_nr = queue_nr;489 heap_array[pos].ordinal = ordinal;490}491 492int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr,493 u64 ordinal)494{495 struct auxtrace_heap_item *heap_array;496 497 if (queue_nr >= heap->heap_sz) {498 unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES;499 500 while (heap_sz <= queue_nr)501 heap_sz <<= 1;502 heap_array = realloc(heap->heap_array,503 heap_sz * sizeof(struct auxtrace_heap_item));504 if (!heap_array)505 return -ENOMEM;506 heap->heap_array = heap_array;507 heap->heap_sz = heap_sz;508 }509 510 auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal);511 512 return 0;513}514 515void auxtrace_heap__free(struct auxtrace_heap *heap)516{517 zfree(&heap->heap_array);518 heap->heap_cnt = 0;519 heap->heap_sz = 0;520}521 522void auxtrace_heap__pop(struct auxtrace_heap *heap)523{524 unsigned int pos, last, heap_cnt = heap->heap_cnt;525 struct auxtrace_heap_item *heap_array;526 527 if (!heap_cnt)528 return;529 530 heap->heap_cnt -= 1;531 532 heap_array = heap->heap_array;533 534 pos = 0;535 while (1) {536 unsigned int left, right;537 538 left = (pos << 1) + 1;539 if (left >= heap_cnt)540 break;541 right = left + 1;542 if (right >= heap_cnt) {543 heap_array[pos] = heap_array[left];544 return;545 }546 if (heap_array[left].ordinal < heap_array[right].ordinal) {547 heap_array[pos] = heap_array[left];548 pos = left;549 } else {550 heap_array[pos] = heap_array[right];551 pos = right;552 }553 }554 555 last = heap_cnt - 1;556 auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr,557 heap_array[last].ordinal);558}559 560size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr,561 struct evlist *evlist)562{563 if (itr)564 return itr->info_priv_size(itr, evlist);565 return 0;566}567 568static int auxtrace_not_supported(void)569{570 pr_err("AUX area tracing is not supported on this architecture\n");571 return -EINVAL;572}573 574int auxtrace_record__info_fill(struct auxtrace_record *itr,575 struct perf_session *session,576 struct perf_record_auxtrace_info *auxtrace_info,577 size_t priv_size)578{579 if (itr)580 return itr->info_fill(itr, session, auxtrace_info, priv_size);581 return auxtrace_not_supported();582}583 584void auxtrace_record__free(struct auxtrace_record *itr)585{586 if (itr)587 itr->free(itr);588}589 590int auxtrace_record__snapshot_start(struct auxtrace_record *itr)591{592 if (itr && itr->snapshot_start)593 return itr->snapshot_start(itr);594 return 0;595}596 597int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit)598{599 if (!on_exit && itr && itr->snapshot_finish)600 return itr->snapshot_finish(itr);601 return 0;602}603 604int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx,605 struct auxtrace_mmap *mm,606 unsigned char *data, u64 *head, u64 *old)607{608 if (itr && itr->find_snapshot)609 return itr->find_snapshot(itr, idx, mm, data, head, old);610 return 0;611}612 613int auxtrace_record__options(struct auxtrace_record *itr,614 struct evlist *evlist,615 struct record_opts *opts)616{617 if (itr) {618 itr->evlist = evlist;619 return itr->recording_options(itr, evlist, opts);620 }621 return 0;622}623 624u64 auxtrace_record__reference(struct auxtrace_record *itr)625{626 if (itr)627 return itr->reference(itr);628 return 0;629}630 631int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,632 struct record_opts *opts, const char *str)633{634 if (!str)635 return 0;636 637 /* PMU-agnostic options */638 switch (*str) {639 case 'e':640 opts->auxtrace_snapshot_on_exit = true;641 str++;642 break;643 default:644 break;645 }646 647 if (itr && itr->parse_snapshot_options)648 return itr->parse_snapshot_options(itr, opts, str);649 650 pr_err("No AUX area tracing to snapshot\n");651 return -EINVAL;652}653 654static int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx)655{656 bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus);657 658 if (per_cpu_mmaps) {659 struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx);660 int cpu_map_idx = perf_cpu_map__idx(evsel->core.cpus, evlist_cpu);661 662 if (cpu_map_idx == -1)663 return -EINVAL;664 return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx);665 }666 667 return perf_evsel__enable_thread(&evsel->core, idx);668}669 670int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx)671{672 struct evsel *evsel;673 674 if (!itr->evlist)675 return -EINVAL;676 677 evlist__for_each_entry(itr->evlist, evsel) {678 if (evsel__is_aux_event(evsel)) {679 if (evsel->disabled)680 return 0;681 return evlist__enable_event_idx(itr->evlist, evsel, idx);682 }683 }684 return -EINVAL;685}686 687/*688 * Event record size is 16-bit which results in a maximum size of about 64KiB.689 * Allow about 4KiB for the rest of the sample record, to give a maximum690 * AUX area sample size of 60KiB.691 */692#define MAX_AUX_SAMPLE_SIZE (60 * 1024)693 694/* Arbitrary default size if no other default provided */695#define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024)696 697static int auxtrace_validate_aux_sample_size(struct evlist *evlist,698 struct record_opts *opts)699{700 struct evsel *evsel;701 bool has_aux_leader = false;702 u32 sz;703 704 evlist__for_each_entry(evlist, evsel) {705 sz = evsel->core.attr.aux_sample_size;706 if (evsel__is_group_leader(evsel)) {707 has_aux_leader = evsel__is_aux_event(evsel);708 if (sz) {709 if (has_aux_leader)710 pr_err("Cannot add AUX area sampling to an AUX area event\n");711 else712 pr_err("Cannot add AUX area sampling to a group leader\n");713 return -EINVAL;714 }715 }716 if (sz > MAX_AUX_SAMPLE_SIZE) {717 pr_err("AUX area sample size %u too big, max. %d\n",718 sz, MAX_AUX_SAMPLE_SIZE);719 return -EINVAL;720 }721 if (sz) {722 if (!has_aux_leader) {723 pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n");724 return -EINVAL;725 }726 evsel__set_sample_bit(evsel, AUX);727 opts->auxtrace_sample_mode = true;728 } else {729 evsel__reset_sample_bit(evsel, AUX);730 }731 }732 733 if (!opts->auxtrace_sample_mode) {734 pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n");735 return -EINVAL;736 }737 738 if (!perf_can_aux_sample()) {739 pr_err("AUX area sampling is not supported by kernel\n");740 return -EINVAL;741 }742 743 return 0;744}745 746int auxtrace_parse_sample_options(struct auxtrace_record *itr,747 struct evlist *evlist,748 struct record_opts *opts, const char *str)749{750 struct evsel_config_term *term;751 struct evsel *aux_evsel;752 bool has_aux_sample_size = false;753 bool has_aux_leader = false;754 struct evsel *evsel;755 char *endptr;756 unsigned long sz;757 758 if (!str)759 goto no_opt;760 761 if (!itr) {762 pr_err("No AUX area event to sample\n");763 return -EINVAL;764 }765 766 sz = strtoul(str, &endptr, 0);767 if (*endptr || sz > UINT_MAX) {768 pr_err("Bad AUX area sampling option: '%s'\n", str);769 return -EINVAL;770 }771 772 if (!sz)773 sz = itr->default_aux_sample_size;774 775 if (!sz)776 sz = DEFAULT_AUX_SAMPLE_SIZE;777 778 /* Set aux_sample_size based on --aux-sample option */779 evlist__for_each_entry(evlist, evsel) {780 if (evsel__is_group_leader(evsel)) {781 has_aux_leader = evsel__is_aux_event(evsel);782 } else if (has_aux_leader) {783 evsel->core.attr.aux_sample_size = sz;784 }785 }786no_opt:787 aux_evsel = NULL;788 /* Override with aux_sample_size from config term */789 evlist__for_each_entry(evlist, evsel) {790 if (evsel__is_aux_event(evsel))791 aux_evsel = evsel;792 term = evsel__get_config_term(evsel, AUX_SAMPLE_SIZE);793 if (term) {794 has_aux_sample_size = true;795 evsel->core.attr.aux_sample_size = term->val.aux_sample_size;796 /* If possible, group with the AUX event */797 if (aux_evsel && evsel->core.attr.aux_sample_size)798 evlist__regroup(evlist, aux_evsel, evsel);799 }800 }801 802 if (!str && !has_aux_sample_size)803 return 0;804 805 if (!itr) {806 pr_err("No AUX area event to sample\n");807 return -EINVAL;808 }809 810 return auxtrace_validate_aux_sample_size(evlist, opts);811}812 813void auxtrace_regroup_aux_output(struct evlist *evlist)814{815 struct evsel *evsel, *aux_evsel = NULL;816 struct evsel_config_term *term;817 818 evlist__for_each_entry(evlist, evsel) {819 if (evsel__is_aux_event(evsel))820 aux_evsel = evsel;821 term = evsel__get_config_term(evsel, AUX_OUTPUT);822 /* If possible, group with the AUX event */823 if (term && aux_evsel)824 evlist__regroup(evlist, aux_evsel, evsel);825 }826}827 828struct auxtrace_record *__weak829auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err)830{831 *err = 0;832 return NULL;833}834 835static int auxtrace_index__alloc(struct list_head *head)836{837 struct auxtrace_index *auxtrace_index;838 839 auxtrace_index = malloc(sizeof(struct auxtrace_index));840 if (!auxtrace_index)841 return -ENOMEM;842 843 auxtrace_index->nr = 0;844 INIT_LIST_HEAD(&auxtrace_index->list);845 846 list_add_tail(&auxtrace_index->list, head);847 848 return 0;849}850 851void auxtrace_index__free(struct list_head *head)852{853 struct auxtrace_index *auxtrace_index, *n;854 855 list_for_each_entry_safe(auxtrace_index, n, head, list) {856 list_del_init(&auxtrace_index->list);857 free(auxtrace_index);858 }859}860 861static struct auxtrace_index *auxtrace_index__last(struct list_head *head)862{863 struct auxtrace_index *auxtrace_index;864 int err;865 866 if (list_empty(head)) {867 err = auxtrace_index__alloc(head);868 if (err)869 return NULL;870 }871 872 auxtrace_index = list_entry(head->prev, struct auxtrace_index, list);873 874 if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) {875 err = auxtrace_index__alloc(head);876 if (err)877 return NULL;878 auxtrace_index = list_entry(head->prev, struct auxtrace_index,879 list);880 }881 882 return auxtrace_index;883}884 885int auxtrace_index__auxtrace_event(struct list_head *head,886 union perf_event *event, off_t file_offset)887{888 struct auxtrace_index *auxtrace_index;889 size_t nr;890 891 auxtrace_index = auxtrace_index__last(head);892 if (!auxtrace_index)893 return -ENOMEM;894 895 nr = auxtrace_index->nr;896 auxtrace_index->entries[nr].file_offset = file_offset;897 auxtrace_index->entries[nr].sz = event->header.size;898 auxtrace_index->nr += 1;899 900 return 0;901}902 903static int auxtrace_index__do_write(int fd,904 struct auxtrace_index *auxtrace_index)905{906 struct auxtrace_index_entry ent;907 size_t i;908 909 for (i = 0; i < auxtrace_index->nr; i++) {910 ent.file_offset = auxtrace_index->entries[i].file_offset;911 ent.sz = auxtrace_index->entries[i].sz;912 if (writen(fd, &ent, sizeof(ent)) != sizeof(ent))913 return -errno;914 }915 return 0;916}917 918int auxtrace_index__write(int fd, struct list_head *head)919{920 struct auxtrace_index *auxtrace_index;921 u64 total = 0;922 int err;923 924 list_for_each_entry(auxtrace_index, head, list)925 total += auxtrace_index->nr;926 927 if (writen(fd, &total, sizeof(total)) != sizeof(total))928 return -errno;929 930 list_for_each_entry(auxtrace_index, head, list) {931 err = auxtrace_index__do_write(fd, auxtrace_index);932 if (err)933 return err;934 }935 936 return 0;937}938 939static int auxtrace_index__process_entry(int fd, struct list_head *head,940 bool needs_swap)941{942 struct auxtrace_index *auxtrace_index;943 struct auxtrace_index_entry ent;944 size_t nr;945 946 if (readn(fd, &ent, sizeof(ent)) != sizeof(ent))947 return -1;948 949 auxtrace_index = auxtrace_index__last(head);950 if (!auxtrace_index)951 return -1;952 953 nr = auxtrace_index->nr;954 if (needs_swap) {955 auxtrace_index->entries[nr].file_offset =956 bswap_64(ent.file_offset);957 auxtrace_index->entries[nr].sz = bswap_64(ent.sz);958 } else {959 auxtrace_index->entries[nr].file_offset = ent.file_offset;960 auxtrace_index->entries[nr].sz = ent.sz;961 }962 963 auxtrace_index->nr = nr + 1;964 965 return 0;966}967 968int auxtrace_index__process(int fd, u64 size, struct perf_session *session,969 bool needs_swap)970{971 struct list_head *head = &session->auxtrace_index;972 u64 nr;973 974 if (readn(fd, &nr, sizeof(u64)) != sizeof(u64))975 return -1;976 977 if (needs_swap)978 nr = bswap_64(nr);979 980 if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size)981 return -1;982 983 while (nr--) {984 int err;985 986 err = auxtrace_index__process_entry(fd, head, needs_swap);987 if (err)988 return -1;989 }990 991 return 0;992}993 994static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues,995 struct perf_session *session,996 struct auxtrace_index_entry *ent)997{998 return auxtrace_queues__add_indexed_event(queues, session,999 ent->file_offset, ent->sz);1000}1001 1002int auxtrace_queues__process_index(struct auxtrace_queues *queues,1003 struct perf_session *session)1004{1005 struct auxtrace_index *auxtrace_index;1006 struct auxtrace_index_entry *ent;1007 size_t i;1008 int err;1009 1010 if (auxtrace__dont_decode(session))1011 return 0;1012 1013 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {1014 for (i = 0; i < auxtrace_index->nr; i++) {1015 ent = &auxtrace_index->entries[i];1016 err = auxtrace_queues__process_index_entry(queues,1017 session,1018 ent);1019 if (err)1020 return err;1021 }1022 }1023 return 0;1024}1025 1026struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue,1027 struct auxtrace_buffer *buffer)1028{1029 if (buffer) {1030 if (list_is_last(&buffer->list, &queue->head))1031 return NULL;1032 return list_entry(buffer->list.next, struct auxtrace_buffer,1033 list);1034 } else {1035 if (list_empty(&queue->head))1036 return NULL;1037 return list_entry(queue->head.next, struct auxtrace_buffer,1038 list);1039 }1040}1041 1042struct auxtrace_queue *auxtrace_queues__sample_queue(struct auxtrace_queues *queues,1043 struct perf_sample *sample,1044 struct perf_session *session)1045{1046 struct perf_sample_id *sid;1047 unsigned int idx;1048 u64 id;1049 1050 id = sample->id;1051 if (!id)1052 return NULL;1053 1054 sid = evlist__id2sid(session->evlist, id);1055 if (!sid)1056 return NULL;1057 1058 idx = sid->idx;1059 1060 if (idx >= queues->nr_queues)1061 return NULL;1062 1063 return &queues->queue_array[idx];1064}1065 1066int auxtrace_queues__add_sample(struct auxtrace_queues *queues,1067 struct perf_session *session,1068 struct perf_sample *sample, u64 data_offset,1069 u64 reference)1070{1071 struct auxtrace_buffer buffer = {1072 .pid = -1,1073 .data_offset = data_offset,1074 .reference = reference,1075 .size = sample->aux_sample.size,1076 };1077 struct perf_sample_id *sid;1078 u64 id = sample->id;1079 unsigned int idx;1080 1081 if (!id)1082 return -EINVAL;1083 1084 sid = evlist__id2sid(session->evlist, id);1085 if (!sid)1086 return -ENOENT;1087 1088 idx = sid->idx;1089 buffer.tid = sid->tid;1090 buffer.cpu = sid->cpu;1091 1092 return auxtrace_queues__add_buffer(queues, session, idx, &buffer, NULL);1093}1094 1095struct queue_data {1096 bool samples;1097 bool events;1098};1099 1100static int auxtrace_queue_data_cb(struct perf_session *session,1101 union perf_event *event, u64 offset,1102 void *data)1103{1104 struct queue_data *qd = data;1105 struct perf_sample sample;1106 int err;1107 1108 if (qd->events && event->header.type == PERF_RECORD_AUXTRACE) {1109 if (event->header.size < sizeof(struct perf_record_auxtrace))1110 return -EINVAL;1111 offset += event->header.size;1112 return session->auxtrace->queue_data(session, NULL, event,1113 offset);1114 }1115 1116 if (!qd->samples || event->header.type != PERF_RECORD_SAMPLE)1117 return 0;1118 1119 err = evlist__parse_sample(session->evlist, event, &sample);1120 if (err)1121 return err;1122 1123 if (!sample.aux_sample.size)1124 return 0;1125 1126 offset += sample.aux_sample.data - (void *)event;1127 1128 return session->auxtrace->queue_data(session, &sample, NULL, offset);1129}1130 1131int auxtrace_queue_data(struct perf_session *session, bool samples, bool events)1132{1133 struct queue_data qd = {1134 .samples = samples,1135 .events = events,1136 };1137 1138 if (auxtrace__dont_decode(session))1139 return 0;1140 1141 if (perf_data__is_pipe(session->data))1142 return 0;1143 1144 if (!session->auxtrace || !session->auxtrace->queue_data)1145 return -EINVAL;1146 1147 return perf_session__peek_events(session, session->header.data_offset,1148 session->header.data_size,1149 auxtrace_queue_data_cb, &qd);1150}1151 1152void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw)1153{1154 int prot = rw ? PROT_READ | PROT_WRITE : PROT_READ;1155 size_t adj = buffer->data_offset & (page_size - 1);1156 size_t size = buffer->size + adj;1157 off_t file_offset = buffer->data_offset - adj;1158 void *addr;1159 1160 if (buffer->data)1161 return buffer->data;1162 1163 addr = mmap(NULL, size, prot, MAP_SHARED, fd, file_offset);1164 if (addr == MAP_FAILED)1165 return NULL;1166 1167 buffer->mmap_addr = addr;1168 buffer->mmap_size = size;1169 1170 buffer->data = addr + adj;1171 1172 return buffer->data;1173}1174 1175void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer)1176{1177 if (!buffer->data || !buffer->mmap_addr)1178 return;1179 munmap(buffer->mmap_addr, buffer->mmap_size);1180 buffer->mmap_addr = NULL;1181 buffer->mmap_size = 0;1182 buffer->data = NULL;1183 buffer->use_data = NULL;1184}1185 1186void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer)1187{1188 auxtrace_buffer__put_data(buffer);1189 if (buffer->data_needs_freeing) {1190 buffer->data_needs_freeing = false;1191 zfree(&buffer->data);1192 buffer->use_data = NULL;1193 buffer->size = 0;1194 }1195}1196 1197void auxtrace_buffer__free(struct auxtrace_buffer *buffer)1198{1199 auxtrace_buffer__drop_data(buffer);1200 free(buffer);1201}1202 1203void auxtrace_synth_guest_error(struct perf_record_auxtrace_error *auxtrace_error, int type,1204 int code, int cpu, pid_t pid, pid_t tid, u64 ip,1205 const char *msg, u64 timestamp,1206 pid_t machine_pid, int vcpu)1207{1208 size_t size;1209 1210 memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error));1211 1212 auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR;1213 auxtrace_error->type = type;1214 auxtrace_error->code = code;1215 auxtrace_error->cpu = cpu;1216 auxtrace_error->pid = pid;1217 auxtrace_error->tid = tid;1218 auxtrace_error->fmt = 1;1219 auxtrace_error->ip = ip;1220 auxtrace_error->time = timestamp;1221 strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG);1222 if (machine_pid) {1223 auxtrace_error->fmt = 2;1224 auxtrace_error->machine_pid = machine_pid;1225 auxtrace_error->vcpu = vcpu;1226 size = sizeof(*auxtrace_error);1227 } else {1228 size = (void *)auxtrace_error->msg - (void *)auxtrace_error +1229 strlen(auxtrace_error->msg) + 1;1230 }1231 auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64));1232}1233 1234void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type,1235 int code, int cpu, pid_t pid, pid_t tid, u64 ip,1236 const char *msg, u64 timestamp)1237{1238 auxtrace_synth_guest_error(auxtrace_error, type, code, cpu, pid, tid,1239 ip, msg, timestamp, 0, -1);1240}1241 1242int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr,1243 const struct perf_tool *tool,1244 struct perf_session *session,1245 perf_event__handler_t process)1246{1247 union perf_event *ev;1248 size_t priv_size;1249 int err;1250 1251 pr_debug2("Synthesizing auxtrace information\n");1252 priv_size = auxtrace_record__info_priv_size(itr, session->evlist);1253 ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size);1254 if (!ev)1255 return -ENOMEM;1256 1257 ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO;1258 ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) +1259 priv_size;1260 err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info,1261 priv_size);1262 if (err)1263 goto out_free;1264 1265 err = process(tool, ev, NULL, NULL);1266out_free:1267 free(ev);1268 return err;1269}1270 1271static void unleader_evsel(struct evlist *evlist, struct evsel *leader)1272{1273 struct evsel *new_leader = NULL;1274 struct evsel *evsel;1275 1276 /* Find new leader for the group */1277 evlist__for_each_entry(evlist, evsel) {1278 if (!evsel__has_leader(evsel, leader) || evsel == leader)1279 continue;1280 if (!new_leader)1281 new_leader = evsel;1282 evsel__set_leader(evsel, new_leader);1283 }1284 1285 /* Update group information */1286 if (new_leader) {1287 zfree(&new_leader->group_name);1288 new_leader->group_name = leader->group_name;1289 leader->group_name = NULL;1290 1291 new_leader->core.nr_members = leader->core.nr_members - 1;1292 leader->core.nr_members = 1;1293 }1294}1295 1296static void unleader_auxtrace(struct perf_session *session)1297{1298 struct evsel *evsel;1299 1300 evlist__for_each_entry(session->evlist, evsel) {1301 if (auxtrace__evsel_is_auxtrace(session, evsel) &&1302 evsel__is_group_leader(evsel)) {1303 unleader_evsel(session->evlist, evsel);1304 }1305 }1306}1307 1308int perf_event__process_auxtrace_info(struct perf_session *session,1309 union perf_event *event)1310{1311 enum auxtrace_type type = event->auxtrace_info.type;1312 int err;1313 1314 if (dump_trace)1315 fprintf(stdout, " type: %u\n", type);1316 1317 switch (type) {1318 case PERF_AUXTRACE_INTEL_PT:1319 err = intel_pt_process_auxtrace_info(event, session);1320 break;1321 case PERF_AUXTRACE_INTEL_BTS:1322 err = intel_bts_process_auxtrace_info(event, session);1323 break;1324 case PERF_AUXTRACE_ARM_SPE:1325 err = arm_spe_process_auxtrace_info(event, session);1326 break;1327 case PERF_AUXTRACE_CS_ETM:1328 err = cs_etm__process_auxtrace_info(event, session);1329 break;1330 case PERF_AUXTRACE_S390_CPUMSF:1331 err = s390_cpumsf_process_auxtrace_info(event, session);1332 break;1333 case PERF_AUXTRACE_HISI_PTT:1334 err = hisi_ptt_process_auxtrace_info(event, session);1335 break;1336 case PERF_AUXTRACE_UNKNOWN:1337 default:1338 return -EINVAL;1339 }1340 1341 if (err)1342 return err;1343 1344 unleader_auxtrace(session);1345 1346 return 0;1347}1348 1349s64 perf_event__process_auxtrace(struct perf_session *session,1350 union perf_event *event)1351{1352 s64 err;1353 1354 if (dump_trace)1355 fprintf(stdout, " size: %#"PRI_lx64" offset: %#"PRI_lx64" ref: %#"PRI_lx64" idx: %u tid: %d cpu: %d\n",1356 event->auxtrace.size, event->auxtrace.offset,1357 event->auxtrace.reference, event->auxtrace.idx,1358 event->auxtrace.tid, event->auxtrace.cpu);1359 1360 if (auxtrace__dont_decode(session))1361 return event->auxtrace.size;1362 1363 if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE)1364 return -EINVAL;1365 1366 err = session->auxtrace->process_auxtrace_event(session, event, session->tool);1367 if (err < 0)1368 return err;1369 1370 return event->auxtrace.size;1371}1372 1373#define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS1374#define PERF_ITRACE_DEFAULT_PERIOD 1000001375#define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 161376#define PERF_ITRACE_MAX_CALLCHAIN_SZ 10241377#define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 641378#define PERF_ITRACE_MAX_LAST_BRANCH_SZ 10241379 1380void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts,1381 bool no_sample)1382{1383 synth_opts->branches = true;1384 synth_opts->transactions = true;1385 synth_opts->ptwrites = true;1386 synth_opts->pwr_events = true;1387 synth_opts->other_events = true;1388 synth_opts->intr_events = true;1389 synth_opts->errors = true;1390 synth_opts->flc = true;1391 synth_opts->llc = true;1392 synth_opts->tlb = true;1393 synth_opts->mem = true;1394 synth_opts->remote_access = true;1395 1396 if (no_sample) {1397 synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS;1398 synth_opts->period = 1;1399 synth_opts->calls = true;1400 } else {1401 synth_opts->instructions = true;1402 synth_opts->cycles = true;1403 synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE;1404 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;1405 }1406 synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;1407 synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;1408 synth_opts->initial_skip = 0;1409}1410 1411static int get_flag(const char **ptr, unsigned int *flags)1412{1413 while (1) {1414 char c = **ptr;1415 1416 if (c >= 'a' && c <= 'z') {1417 *flags |= 1 << (c - 'a');1418 ++*ptr;1419 return 0;1420 } else if (c == ' ') {1421 ++*ptr;1422 continue;1423 } else {1424 return -1;1425 }1426 }1427}1428 1429static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *minus_flags)1430{1431 while (1) {1432 switch (**ptr) {1433 case '+':1434 ++*ptr;1435 if (get_flag(ptr, plus_flags))1436 return -1;1437 break;1438 case '-':1439 ++*ptr;1440 if (get_flag(ptr, minus_flags))1441 return -1;1442 break;1443 case ' ':1444 ++*ptr;1445 break;1446 default:1447 return 0;1448 }1449 }1450}1451 1452#define ITRACE_DFLT_LOG_ON_ERROR_SZ 163841453 1454static unsigned int itrace_log_on_error_size(void)1455{1456 unsigned int sz = 0;1457 1458 perf_config_scan("itrace.debug-log-buffer-size", "%u", &sz);1459 return sz ?: ITRACE_DFLT_LOG_ON_ERROR_SZ;1460}1461 1462/*1463 * Please check tools/perf/Documentation/perf-script.txt for information1464 * about the options parsed here, which is introduced after this cset,1465 * when support in 'perf script' for these options is introduced.1466 */1467int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts,1468 const char *str, int unset)1469{1470 const char *p;1471 char *endptr;1472 bool period_type_set = false;1473 bool period_set = false;1474 bool iy = false;1475 1476 synth_opts->set = true;1477 1478 if (unset) {1479 synth_opts->dont_decode = true;1480 return 0;1481 }1482 1483 if (!str) {1484 itrace_synth_opts__set_default(synth_opts,1485 synth_opts->default_no_sample);1486 return 0;1487 }1488 1489 for (p = str; *p;) {1490 switch (*p++) {1491 case 'i':1492 case 'y':1493 iy = true;1494 if (p[-1] == 'y')1495 synth_opts->cycles = true;1496 else1497 synth_opts->instructions = true;1498 while (*p == ' ' || *p == ',')1499 p += 1;1500 if (isdigit(*p)) {1501 synth_opts->period = strtoull(p, &endptr, 10);1502 period_set = true;1503 p = endptr;1504 while (*p == ' ' || *p == ',')1505 p += 1;1506 switch (*p++) {1507 case 'i':1508 synth_opts->period_type =1509 PERF_ITRACE_PERIOD_INSTRUCTIONS;1510 period_type_set = true;1511 break;1512 case 't':1513 synth_opts->period_type =1514 PERF_ITRACE_PERIOD_TICKS;1515 period_type_set = true;1516 break;1517 case 'm':1518 synth_opts->period *= 1000;1519 /* Fall through */1520 case 'u':1521 synth_opts->period *= 1000;1522 /* Fall through */1523 case 'n':1524 if (*p++ != 's')1525 goto out_err;1526 synth_opts->period_type =1527 PERF_ITRACE_PERIOD_NANOSECS;1528 period_type_set = true;1529 break;1530 case '\0':1531 goto out;1532 default:1533 goto out_err;1534 }1535 }1536 break;1537 case 'b':1538 synth_opts->branches = true;1539 break;1540 case 'x':1541 synth_opts->transactions = true;1542 break;1543 case 'w':1544 synth_opts->ptwrites = true;1545 break;1546 case 'p':1547 synth_opts->pwr_events = true;1548 break;1549 case 'o':1550 synth_opts->other_events = true;1551 break;1552 case 'I':1553 synth_opts->intr_events = true;1554 break;1555 case 'e':1556 synth_opts->errors = true;1557 if (get_flags(&p, &synth_opts->error_plus_flags,1558 &synth_opts->error_minus_flags))1559 goto out_err;1560 break;1561 case 'd':1562 synth_opts->log = true;1563 if (get_flags(&p, &synth_opts->log_plus_flags,1564 &synth_opts->log_minus_flags))1565 goto out_err;1566 if (synth_opts->log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR)1567 synth_opts->log_on_error_size = itrace_log_on_error_size();1568 break;1569 case 'c':1570 synth_opts->branches = true;1571 synth_opts->calls = true;1572 break;1573 case 'r':1574 synth_opts->branches = true;1575 synth_opts->returns = true;1576 break;1577 case 'G':1578 case 'g':1579 if (p[-1] == 'G')1580 synth_opts->add_callchain = true;1581 else1582 synth_opts->callchain = true;1583 synth_opts->callchain_sz =1584 PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;1585 while (*p == ' ' || *p == ',')1586 p += 1;1587 if (isdigit(*p)) {1588 unsigned int val;1589 1590 val = strtoul(p, &endptr, 10);1591 p = endptr;1592 if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ)1593 goto out_err;1594 synth_opts->callchain_sz = val;1595 }1596 break;1597 case 'L':1598 case 'l':1599 if (p[-1] == 'L')1600 synth_opts->add_last_branch = true;1601 else1602 synth_opts->last_branch = true;1603 synth_opts->last_branch_sz =1604 PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;1605 while (*p == ' ' || *p == ',')1606 p += 1;1607 if (isdigit(*p)) {1608 unsigned int val;1609 1610 val = strtoul(p, &endptr, 10);1611 p = endptr;1612 if (!val ||1613 val > PERF_ITRACE_MAX_LAST_BRANCH_SZ)1614 goto out_err;1615 synth_opts->last_branch_sz = val;1616 }1617 break;1618 case 's':1619 synth_opts->initial_skip = strtoul(p, &endptr, 10);1620 if (p == endptr)1621 goto out_err;1622 p = endptr;1623 break;1624 case 'f':1625 synth_opts->flc = true;1626 break;1627 case 'm':1628 synth_opts->llc = true;1629 break;1630 case 't':1631 synth_opts->tlb = true;1632 break;1633 case 'a':1634 synth_opts->remote_access = true;1635 break;1636 case 'M':1637 synth_opts->mem = true;1638 break;1639 case 'q':1640 synth_opts->quick += 1;1641 break;1642 case 'A':1643 synth_opts->approx_ipc = true;1644 break;1645 case 'Z':1646 synth_opts->timeless_decoding = true;1647 break;1648 case 'T':1649 synth_opts->use_timestamp = true;1650 break;1651 case ' ':1652 case ',':1653 break;1654 default:1655 goto out_err;1656 }1657 }1658out:1659 if (iy) {1660 if (!period_type_set)1661 synth_opts->period_type =1662 PERF_ITRACE_DEFAULT_PERIOD_TYPE;1663 if (!period_set)1664 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;1665 }1666 1667 return 0;1668 1669out_err:1670 pr_err("Bad Instruction Tracing options '%s'\n", str);1671 return -EINVAL;1672}1673 1674int itrace_parse_synth_opts(const struct option *opt, const char *str, int unset)1675{1676 return itrace_do_parse_synth_opts(opt->value, str, unset);1677}1678 1679static const char * const auxtrace_error_type_name[] = {1680 [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace",1681};1682 1683static const char *auxtrace_error_name(int type)1684{1685 const char *error_type_name = NULL;1686 1687 if (type < PERF_AUXTRACE_ERROR_MAX)1688 error_type_name = auxtrace_error_type_name[type];1689 if (!error_type_name)1690 error_type_name = "unknown AUX";1691 return error_type_name;1692}1693 1694size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp)1695{1696 struct perf_record_auxtrace_error *e = &event->auxtrace_error;1697 unsigned long long nsecs = e->time;1698 const char *msg = e->msg;1699 int ret;1700 1701 ret = fprintf(fp, " %s error type %u",1702 auxtrace_error_name(e->type), e->type);1703 1704 if (e->fmt && nsecs) {1705 unsigned long secs = nsecs / NSEC_PER_SEC;1706 1707 nsecs -= secs * NSEC_PER_SEC;1708 ret += fprintf(fp, " time %lu.%09llu", secs, nsecs);1709 } else {1710 ret += fprintf(fp, " time 0");1711 }1712 1713 if (!e->fmt)1714 msg = (const char *)&e->time;1715 1716 if (e->fmt >= 2 && e->machine_pid)1717 ret += fprintf(fp, " machine_pid %d vcpu %d", e->machine_pid, e->vcpu);1718 1719 ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n",1720 e->cpu, e->pid, e->tid, e->ip, e->code, msg);1721 return ret;1722}1723 1724void perf_session__auxtrace_error_inc(struct perf_session *session,1725 union perf_event *event)1726{1727 struct perf_record_auxtrace_error *e = &event->auxtrace_error;1728 1729 if (e->type < PERF_AUXTRACE_ERROR_MAX)1730 session->evlist->stats.nr_auxtrace_errors[e->type] += 1;1731}1732 1733void events_stats__auxtrace_error_warn(const struct events_stats *stats)1734{1735 int i;1736 1737 for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) {1738 if (!stats->nr_auxtrace_errors[i])1739 continue;1740 ui__warning("%u %s errors\n",1741 stats->nr_auxtrace_errors[i],1742 auxtrace_error_name(i));1743 }1744}1745 1746int perf_event__process_auxtrace_error(struct perf_session *session,1747 union perf_event *event)1748{1749 if (auxtrace__dont_decode(session))1750 return 0;1751 1752 perf_event__fprintf_auxtrace_error(event, stdout);1753 return 0;1754}1755 1756/*1757 * In the compat mode kernel runs in 64-bit and perf tool runs in 32-bit mode,1758 * 32-bit perf tool cannot access 64-bit value atomically, which might lead to1759 * the issues caused by the below sequence on multiple CPUs: when perf tool1760 * accesses either the load operation or the store operation for 64-bit value,1761 * on some architectures the operation is divided into two instructions, one1762 * is for accessing the low 32-bit value and another is for the high 32-bit;1763 * thus these two user operations can give the kernel chances to access the1764 * 64-bit value, and thus leads to the unexpected load values.1765 *1766 * kernel (64-bit) user (32-bit)1767 *1768 * if (LOAD ->aux_tail) { --, LOAD ->aux_head_lo1769 * STORE $aux_data | ,--->1770 * FLUSH $aux_data | | LOAD ->aux_head_hi1771 * STORE ->aux_head --|-------` smp_rmb()1772 * } | LOAD $data1773 * | smp_mb()1774 * | STORE ->aux_tail_lo1775 * `----------->1776 * STORE ->aux_tail_hi1777 *1778 * For this reason, it's impossible for the perf tool to work correctly when1779 * the AUX head or tail is bigger than 4GB (more than 32 bits length); and we1780 * can not simply limit the AUX ring buffer to less than 4GB, the reason is1781 * the pointers can be increased monotonically, whatever the buffer size it is,1782 * at the end the head and tail can be bigger than 4GB and carry out to the1783 * high 32-bit.1784 *1785 * To mitigate the issues and improve the user experience, we can allow the1786 * perf tool working in certain conditions and bail out with error if detect1787 * any overflow cannot be handled.1788 *1789 * For reading the AUX head, it reads out the values for three times, and1790 * compares the high 4 bytes of the values between the first time and the last1791 * time, if there has no change for high 4 bytes injected by the kernel during1792 * the user reading sequence, it's safe for use the second value.1793 *1794 * When compat_auxtrace_mmap__write_tail() detects any carrying in the high1795 * 32 bits, it means there have two store operations in user space and it cannot1796 * promise the atomicity for 64-bit write, so return '-1' in this case to tell1797 * the caller an overflow error has happened.1798 */1799u64 __weak compat_auxtrace_mmap__read_head(struct auxtrace_mmap *mm)1800{1801 struct perf_event_mmap_page *pc = mm->userpg;1802 u64 first, second, last;1803 u64 mask = (u64)(UINT32_MAX) << 32;1804 1805 do {1806 first = READ_ONCE(pc->aux_head);1807 /* Ensure all reads are done after we read the head */1808 smp_rmb();1809 second = READ_ONCE(pc->aux_head);1810 /* Ensure all reads are done after we read the head */1811 smp_rmb();1812 last = READ_ONCE(pc->aux_head);1813 } while ((first & mask) != (last & mask));1814 1815 return second;1816}1817 1818int __weak compat_auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail)1819{1820 struct perf_event_mmap_page *pc = mm->userpg;1821 u64 mask = (u64)(UINT32_MAX) << 32;1822 1823 if (tail & mask)1824 return -1;1825 1826 /* Ensure all reads are done before we write the tail out */1827 smp_mb();1828 WRITE_ONCE(pc->aux_tail, tail);1829 return 0;1830}1831 1832static int __auxtrace_mmap__read(struct mmap *map,1833 struct auxtrace_record *itr,1834 const struct perf_tool *tool, process_auxtrace_t fn,1835 bool snapshot, size_t snapshot_size)1836{1837 struct auxtrace_mmap *mm = &map->auxtrace_mmap;1838 u64 head, old = mm->prev, offset, ref;1839 unsigned char *data = mm->base;1840 size_t size, head_off, old_off, len1, len2, padding;1841 union perf_event ev;1842 void *data1, *data2;1843 int kernel_is_64_bit = perf_env__kernel_is_64_bit(evsel__env(NULL));1844 1845 head = auxtrace_mmap__read_head(mm, kernel_is_64_bit);1846 1847 if (snapshot &&1848 auxtrace_record__find_snapshot(itr, mm->idx, mm, data, &head, &old))1849 return -1;1850 1851 if (old == head)1852 return 0;1853 1854 pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n",1855 mm->idx, old, head, head - old);1856 1857 if (mm->mask) {1858 head_off = head & mm->mask;1859 old_off = old & mm->mask;1860 } else {1861 head_off = head % mm->len;1862 old_off = old % mm->len;1863 }1864 1865 if (head_off > old_off)1866 size = head_off - old_off;1867 else1868 size = mm->len - (old_off - head_off);1869 1870 if (snapshot && size > snapshot_size)1871 size = snapshot_size;1872 1873 ref = auxtrace_record__reference(itr);1874 1875 if (head > old || size <= head || mm->mask) {1876 offset = head - size;1877 } else {1878 /*1879 * When the buffer size is not a power of 2, 'head' wraps at the1880 * highest multiple of the buffer size, so we have to subtract1881 * the remainder here.1882 */1883 u64 rem = (0ULL - mm->len) % mm->len;1884 1885 offset = head - size - rem;1886 }1887 1888 if (size > head_off) {1889 len1 = size - head_off;1890 data1 = &data[mm->len - len1];1891 len2 = head_off;1892 data2 = &data[0];1893 } else {1894 len1 = size;1895 data1 = &data[head_off - len1];1896 len2 = 0;1897 data2 = NULL;1898 }1899 1900 if (itr->alignment) {1901 unsigned int unwanted = len1 % itr->alignment;1902 1903 len1 -= unwanted;1904 size -= unwanted;1905 }1906 1907 /* padding must be written by fn() e.g. record__process_auxtrace() */1908 padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1);1909 if (padding)1910 padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding;1911 1912 memset(&ev, 0, sizeof(ev));1913 ev.auxtrace.header.type = PERF_RECORD_AUXTRACE;1914 ev.auxtrace.header.size = sizeof(ev.auxtrace);1915 ev.auxtrace.size = size + padding;1916 ev.auxtrace.offset = offset;1917 ev.auxtrace.reference = ref;1918 ev.auxtrace.idx = mm->idx;1919 ev.auxtrace.tid = mm->tid;1920 ev.auxtrace.cpu = mm->cpu;1921 1922 if (fn(tool, map, &ev, data1, len1, data2, len2))1923 return -1;1924 1925 mm->prev = head;1926 1927 if (!snapshot) {1928 int err;1929 1930 err = auxtrace_mmap__write_tail(mm, head, kernel_is_64_bit);1931 if (err < 0)1932 return err;1933 1934 if (itr->read_finish) {1935 err = itr->read_finish(itr, mm->idx);1936 if (err < 0)1937 return err;1938 }1939 }1940 1941 return 1;1942}1943 1944int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr,1945 const struct perf_tool *tool, process_auxtrace_t fn)1946{1947 return __auxtrace_mmap__read(map, itr, tool, fn, false, 0);1948}1949 1950int auxtrace_mmap__read_snapshot(struct mmap *map,1951 struct auxtrace_record *itr,1952 const struct perf_tool *tool, process_auxtrace_t fn,1953 size_t snapshot_size)1954{1955 return __auxtrace_mmap__read(map, itr, tool, fn, true, snapshot_size);1956}1957 1958/**1959 * struct auxtrace_cache - hash table to implement a cache1960 * @hashtable: the hashtable1961 * @sz: hashtable size (number of hlists)1962 * @entry_size: size of an entry1963 * @limit: limit the number of entries to this maximum, when reached the cache1964 * is dropped and caching begins again with an empty cache1965 * @cnt: current number of entries1966 * @bits: hashtable size (@sz = 2^@bits)1967 */1968struct auxtrace_cache {1969 struct hlist_head *hashtable;1970 size_t sz;1971 size_t entry_size;1972 size_t limit;1973 size_t cnt;1974 unsigned int bits;1975};1976 1977struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size,1978 unsigned int limit_percent)1979{1980 struct auxtrace_cache *c;1981 struct hlist_head *ht;1982 size_t sz, i;1983 1984 c = zalloc(sizeof(struct auxtrace_cache));1985 if (!c)1986 return NULL;1987 1988 sz = 1UL << bits;1989 1990 ht = calloc(sz, sizeof(struct hlist_head));1991 if (!ht)1992 goto out_free;1993 1994 for (i = 0; i < sz; i++)1995 INIT_HLIST_HEAD(&ht[i]);1996 1997 c->hashtable = ht;1998 c->sz = sz;1999 c->entry_size = entry_size;2000 c->limit = (c->sz * limit_percent) / 100;2001 c->bits = bits;2002 2003 return c;2004 2005out_free:2006 free(c);2007 return NULL;2008}2009 2010static void auxtrace_cache__drop(struct auxtrace_cache *c)2011{2012 struct auxtrace_cache_entry *entry;2013 struct hlist_node *tmp;2014 size_t i;2015 2016 if (!c)2017 return;2018 2019 for (i = 0; i < c->sz; i++) {2020 hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) {2021 hlist_del(&entry->hash);2022 auxtrace_cache__free_entry(c, entry);2023 }2024 }2025 2026 c->cnt = 0;2027}2028 2029void auxtrace_cache__free(struct auxtrace_cache *c)2030{2031 if (!c)2032 return;2033 2034 auxtrace_cache__drop(c);2035 zfree(&c->hashtable);2036 free(c);2037}2038 2039void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c)2040{2041 return malloc(c->entry_size);2042}2043 2044void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused,2045 void *entry)2046{2047 free(entry);2048}2049 2050int auxtrace_cache__add(struct auxtrace_cache *c, u32 key,2051 struct auxtrace_cache_entry *entry)2052{2053 if (c->limit && ++c->cnt > c->limit)2054 auxtrace_cache__drop(c);2055 2056 entry->key = key;2057 hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]);2058 2059 return 0;2060}2061 2062static struct auxtrace_cache_entry *auxtrace_cache__rm(struct auxtrace_cache *c,2063 u32 key)2064{2065 struct auxtrace_cache_entry *entry;2066 struct hlist_head *hlist;2067 struct hlist_node *n;2068 2069 if (!c)2070 return NULL;2071 2072 hlist = &c->hashtable[hash_32(key, c->bits)];2073 hlist_for_each_entry_safe(entry, n, hlist, hash) {2074 if (entry->key == key) {2075 hlist_del(&entry->hash);2076 return entry;2077 }2078 }2079 2080 return NULL;2081}2082 2083void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key)2084{2085 struct auxtrace_cache_entry *entry = auxtrace_cache__rm(c, key);2086 2087 auxtrace_cache__free_entry(c, entry);2088}2089 2090void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key)2091{2092 struct auxtrace_cache_entry *entry;2093 struct hlist_head *hlist;2094 2095 if (!c)2096 return NULL;2097 2098 hlist = &c->hashtable[hash_32(key, c->bits)];2099 hlist_for_each_entry(entry, hlist, hash) {2100 if (entry->key == key)2101 return entry;2102 }2103 2104 return NULL;2105}2106 2107static void addr_filter__free_str(struct addr_filter *filt)2108{2109 zfree(&filt->str);2110 filt->action = NULL;2111 filt->sym_from = NULL;2112 filt->sym_to = NULL;2113 filt->filename = NULL;2114}2115 2116static struct addr_filter *addr_filter__new(void)2117{2118 struct addr_filter *filt = zalloc(sizeof(*filt));2119 2120 if (filt)2121 INIT_LIST_HEAD(&filt->list);2122 2123 return filt;2124}2125 2126static void addr_filter__free(struct addr_filter *filt)2127{2128 if (filt)2129 addr_filter__free_str(filt);2130 free(filt);2131}2132 2133static void addr_filters__add(struct addr_filters *filts,2134 struct addr_filter *filt)2135{2136 list_add_tail(&filt->list, &filts->head);2137 filts->cnt += 1;2138}2139 2140static void addr_filters__del(struct addr_filters *filts,2141 struct addr_filter *filt)2142{2143 list_del_init(&filt->list);2144 filts->cnt -= 1;2145}2146 2147void addr_filters__init(struct addr_filters *filts)2148{2149 INIT_LIST_HEAD(&filts->head);2150 filts->cnt = 0;2151}2152 2153void addr_filters__exit(struct addr_filters *filts)2154{2155 struct addr_filter *filt, *n;2156 2157 list_for_each_entry_safe(filt, n, &filts->head, list) {2158 addr_filters__del(filts, filt);2159 addr_filter__free(filt);2160 }2161}2162 2163static int parse_num_or_str(char **inp, u64 *num, const char **str,2164 const char *str_delim)2165{2166 *inp += strspn(*inp, " ");2167 2168 if (isdigit(**inp)) {2169 char *endptr;2170 2171 if (!num)2172 return -EINVAL;2173 errno = 0;2174 *num = strtoull(*inp, &endptr, 0);2175 if (errno)2176 return -errno;2177 if (endptr == *inp)2178 return -EINVAL;2179 *inp = endptr;2180 } else {2181 size_t n;2182 2183 if (!str)2184 return -EINVAL;2185 *inp += strspn(*inp, " ");2186 *str = *inp;2187 n = strcspn(*inp, str_delim);2188 if (!n)2189 return -EINVAL;2190 *inp += n;2191 if (**inp) {2192 **inp = '\0';2193 *inp += 1;2194 }2195 }2196 return 0;2197}2198 2199static int parse_action(struct addr_filter *filt)2200{2201 if (!strcmp(filt->action, "filter")) {2202 filt->start = true;2203 filt->range = true;2204 } else if (!strcmp(filt->action, "start")) {2205 filt->start = true;2206 } else if (!strcmp(filt->action, "stop")) {2207 filt->start = false;2208 } else if (!strcmp(filt->action, "tracestop")) {2209 filt->start = false;2210 filt->range = true;2211 filt->action += 5; /* Change 'tracestop' to 'stop' */2212 } else {2213 return -EINVAL;2214 }2215 return 0;2216}2217 2218static int parse_sym_idx(char **inp, int *idx)2219{2220 *idx = -1;2221 2222 *inp += strspn(*inp, " ");2223 2224 if (**inp != '#')2225 return 0;2226 2227 *inp += 1;2228 2229 if (**inp == 'g' || **inp == 'G') {2230 *inp += 1;2231 *idx = 0;2232 } else {2233 unsigned long num;2234 char *endptr;2235 2236 errno = 0;2237 num = strtoul(*inp, &endptr, 0);2238 if (errno)2239 return -errno;2240 if (endptr == *inp || num > INT_MAX)2241 return -EINVAL;2242 *inp = endptr;2243 *idx = num;2244 }2245 2246 return 0;2247}2248 2249static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx)2250{2251 int err = parse_num_or_str(inp, num, str, " ");2252 2253 if (!err && *str)2254 err = parse_sym_idx(inp, idx);2255 2256 return err;2257}2258 2259static int parse_one_filter(struct addr_filter *filt, const char **filter_inp)2260{2261 char *fstr;2262 int err;2263 2264 filt->str = fstr = strdup(*filter_inp);2265 if (!fstr)2266 return -ENOMEM;2267 2268 err = parse_num_or_str(&fstr, NULL, &filt->action, " ");2269 if (err)2270 goto out_err;2271 2272 err = parse_action(filt);2273 if (err)2274 goto out_err;2275 2276 err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from,2277 &filt->sym_from_idx);2278 if (err)2279 goto out_err;2280 2281 fstr += strspn(fstr, " ");2282 2283 if (*fstr == '/') {2284 fstr += 1;2285 err = parse_addr_size(&fstr, &filt->size, &filt->sym_to,2286 &filt->sym_to_idx);2287 if (err)2288 goto out_err;2289 filt->range = true;2290 }2291 2292 fstr += strspn(fstr, " ");2293 2294 if (*fstr == '@') {2295 fstr += 1;2296 err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,");2297 if (err)2298 goto out_err;2299 }2300 2301 fstr += strspn(fstr, " ,");2302 2303 *filter_inp += fstr - filt->str;2304 2305 return 0;2306 2307out_err:2308 addr_filter__free_str(filt);2309 2310 return err;2311}2312 2313int addr_filters__parse_bare_filter(struct addr_filters *filts,2314 const char *filter)2315{2316 struct addr_filter *filt;2317 const char *fstr = filter;2318 int err;2319 2320 while (*fstr) {2321 filt = addr_filter__new();2322 err = parse_one_filter(filt, &fstr);2323 if (err) {2324 addr_filter__free(filt);2325 addr_filters__exit(filts);2326 return err;2327 }2328 addr_filters__add(filts, filt);2329 }2330 2331 return 0;2332}2333 2334struct sym_args {2335 const char *name;2336 u64 start;2337 u64 size;2338 int idx;2339 int cnt;2340 bool started;2341 bool global;2342 bool selected;2343 bool duplicate;2344 bool near;2345};2346 2347static bool kern_sym_name_match(const char *kname, const char *name)2348{2349 size_t n = strlen(name);2350 2351 return !strcmp(kname, name) ||2352 (!strncmp(kname, name, n) && kname[n] == '\t');2353}2354 2355static bool kern_sym_match(struct sym_args *args, const char *name, char type)2356{2357 /* A function with the same name, and global or the n'th found or any */2358 return kallsyms__is_function(type) &&2359 kern_sym_name_match(name, args->name) &&2360 ((args->global && isupper(type)) ||2361 (args->selected && ++(args->cnt) == args->idx) ||2362 (!args->global && !args->selected));2363}2364 2365static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start)2366{2367 struct sym_args *args = arg;2368 2369 if (args->started) {2370 if (!args->size)2371 args->size = start - args->start;2372 if (args->selected) {2373 if (args->size)2374 return 1;2375 } else if (kern_sym_match(args, name, type)) {2376 args->duplicate = true;2377 return 1;2378 }2379 } else if (kern_sym_match(args, name, type)) {2380 args->started = true;2381 args->start = start;2382 }2383 2384 return 0;2385}2386 2387static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start)2388{2389 struct sym_args *args = arg;2390 2391 if (kern_sym_match(args, name, type)) {2392 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",2393 ++args->cnt, start, type, name);2394 args->near = true;2395 } else if (args->near) {2396 args->near = false;2397 pr_err("\t\twhich is near\t\t%s\n", name);2398 }2399 2400 return 0;2401}2402 2403static int sym_not_found_error(const char *sym_name, int idx)2404{2405 if (idx > 0) {2406 pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n",2407 idx, sym_name);2408 } else if (!idx) {2409 pr_err("Global symbol '%s' not found.\n", sym_name);2410 } else {2411 pr_err("Symbol '%s' not found.\n", sym_name);2412 }2413 pr_err("Note that symbols must be functions.\n");2414 2415 return -EINVAL;2416}2417 2418static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx)2419{2420 struct sym_args args = {2421 .name = sym_name,2422 .idx = idx,2423 .global = !idx,2424 .selected = idx > 0,2425 };2426 int err;2427 2428 *start = 0;2429 *size = 0;2430 2431 err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb);2432 if (err < 0) {2433 pr_err("Failed to parse /proc/kallsyms\n");2434 return err;2435 }2436 2437 if (args.duplicate) {2438 pr_err("Multiple kernel symbols with name '%s'\n", sym_name);2439 args.cnt = 0;2440 kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb);2441 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",2442 sym_name);2443 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");2444 return -EINVAL;2445 }2446 2447 if (!args.started) {2448 pr_err("Kernel symbol lookup: ");2449 return sym_not_found_error(sym_name, idx);2450 }2451 2452 *start = args.start;2453 *size = args.size;2454 2455 return 0;2456}2457 2458static int find_entire_kern_cb(void *arg, const char *name __maybe_unused,2459 char type, u64 start)2460{2461 struct sym_args *args = arg;2462 u64 size;2463 2464 if (!kallsyms__is_function(type))2465 return 0;2466 2467 if (!args->started) {2468 args->started = true;2469 args->start = start;2470 }2471 /* Don't know exactly where the kernel ends, so we add a page */2472 size = round_up(start, page_size) + page_size - args->start;2473 if (size > args->size)2474 args->size = size;2475 2476 return 0;2477}2478 2479static int addr_filter__entire_kernel(struct addr_filter *filt)2480{2481 struct sym_args args = { .started = false };2482 int err;2483 2484 err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb);2485 if (err < 0 || !args.started) {2486 pr_err("Failed to parse /proc/kallsyms\n");2487 return err;2488 }2489 2490 filt->addr = args.start;2491 filt->size = args.size;2492 2493 return 0;2494}2495 2496static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size)2497{2498 if (start + size >= filt->addr)2499 return 0;2500 2501 if (filt->sym_from) {2502 pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n",2503 filt->sym_to, start, filt->sym_from, filt->addr);2504 } else {2505 pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n",2506 filt->sym_to, start, filt->addr);2507 }2508 2509 return -EINVAL;2510}2511 2512static int addr_filter__resolve_kernel_syms(struct addr_filter *filt)2513{2514 bool no_size = false;2515 u64 start, size;2516 int err;2517 2518 if (symbol_conf.kptr_restrict) {2519 pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n");2520 return -EINVAL;2521 }2522 2523 if (filt->sym_from && !strcmp(filt->sym_from, "*"))2524 return addr_filter__entire_kernel(filt);2525 2526 if (filt->sym_from) {2527 err = find_kern_sym(filt->sym_from, &start, &size,2528 filt->sym_from_idx);2529 if (err)2530 return err;2531 filt->addr = start;2532 if (filt->range && !filt->size && !filt->sym_to) {2533 filt->size = size;2534 no_size = !size;2535 }2536 }2537 2538 if (filt->sym_to) {2539 err = find_kern_sym(filt->sym_to, &start, &size,2540 filt->sym_to_idx);2541 if (err)2542 return err;2543 2544 err = check_end_after_start(filt, start, size);2545 if (err)2546 return err;2547 filt->size = start + size - filt->addr;2548 no_size = !size;2549 }2550 2551 /* The very last symbol in kallsyms does not imply a particular size */2552 if (no_size) {2553 pr_err("Cannot determine size of symbol '%s'\n",2554 filt->sym_to ? filt->sym_to : filt->sym_from);2555 return -EINVAL;2556 }2557 2558 return 0;2559}2560 2561static struct dso *load_dso(const char *name)2562{2563 struct map *map;2564 struct dso *dso;2565 2566 map = dso__new_map(name);2567 if (!map)2568 return NULL;2569 2570 if (map__load(map) < 0)2571 pr_err("File '%s' not found or has no symbols.\n", name);2572 2573 dso = dso__get(map__dso(map));2574 2575 map__put(map);2576 2577 return dso;2578}2579 2580static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt,2581 int idx)2582{2583 /* Same name, and global or the n'th found or any */2584 return !arch__compare_symbol_names(name, sym->name) &&2585 ((!idx && sym->binding == STB_GLOBAL) ||2586 (idx > 0 && ++*cnt == idx) ||2587 idx < 0);2588}2589 2590static void print_duplicate_syms(struct dso *dso, const char *sym_name)2591{2592 struct symbol *sym;2593 bool near = false;2594 int cnt = 0;2595 2596 pr_err("Multiple symbols with name '%s'\n", sym_name);2597 2598 sym = dso__first_symbol(dso);2599 while (sym) {2600 if (dso_sym_match(sym, sym_name, &cnt, -1)) {2601 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",2602 ++cnt, sym->start,2603 sym->binding == STB_GLOBAL ? 'g' :2604 sym->binding == STB_LOCAL ? 'l' : 'w',2605 sym->name);2606 near = true;2607 } else if (near) {2608 near = false;2609 pr_err("\t\twhich is near\t\t%s\n", sym->name);2610 }2611 sym = dso__next_symbol(sym);2612 }2613 2614 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",2615 sym_name);2616 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");2617}2618 2619static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start,2620 u64 *size, int idx)2621{2622 struct symbol *sym;2623 int cnt = 0;2624 2625 *start = 0;2626 *size = 0;2627 2628 sym = dso__first_symbol(dso);2629 while (sym) {2630 if (*start) {2631 if (!*size)2632 *size = sym->start - *start;2633 if (idx > 0) {2634 if (*size)2635 return 0;2636 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {2637 print_duplicate_syms(dso, sym_name);2638 return -EINVAL;2639 }2640 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {2641 *start = sym->start;2642 *size = sym->end - sym->start;2643 }2644 sym = dso__next_symbol(sym);2645 }2646 2647 if (!*start)2648 return sym_not_found_error(sym_name, idx);2649 2650 return 0;2651}2652 2653static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso)2654{2655 if (dso__data_file_size(dso, NULL)) {2656 pr_err("Failed to determine filter for %s\nCannot determine file size.\n",2657 filt->filename);2658 return -EINVAL;2659 }2660 2661 filt->addr = 0;2662 filt->size = dso__data(dso)->file_size;2663 2664 return 0;2665}2666 2667static int addr_filter__resolve_syms(struct addr_filter *filt)2668{2669 u64 start, size;2670 struct dso *dso;2671 int err = 0;2672 2673 if (!filt->sym_from && !filt->sym_to)2674 return 0;2675 2676 if (!filt->filename)2677 return addr_filter__resolve_kernel_syms(filt);2678 2679 dso = load_dso(filt->filename);2680 if (!dso) {2681 pr_err("Failed to load symbols from: %s\n", filt->filename);2682 return -EINVAL;2683 }2684 2685 if (filt->sym_from && !strcmp(filt->sym_from, "*")) {2686 err = addr_filter__entire_dso(filt, dso);2687 goto put_dso;2688 }2689 2690 if (filt->sym_from) {2691 err = find_dso_sym(dso, filt->sym_from, &start, &size,2692 filt->sym_from_idx);2693 if (err)2694 goto put_dso;2695 filt->addr = start;2696 if (filt->range && !filt->size && !filt->sym_to)2697 filt->size = size;2698 }2699 2700 if (filt->sym_to) {2701 err = find_dso_sym(dso, filt->sym_to, &start, &size,2702 filt->sym_to_idx);2703 if (err)2704 goto put_dso;2705 2706 err = check_end_after_start(filt, start, size);2707 if (err)2708 return err;2709 2710 filt->size = start + size - filt->addr;2711 }2712 2713put_dso:2714 dso__put(dso);2715 2716 return err;2717}2718 2719static char *addr_filter__to_str(struct addr_filter *filt)2720{2721 char filename_buf[PATH_MAX];2722 const char *at = "";2723 const char *fn = "";2724 char *filter;2725 int err;2726 2727 if (filt->filename) {2728 at = "@";2729 fn = realpath(filt->filename, filename_buf);2730 if (!fn)2731 return NULL;2732 }2733 2734 if (filt->range) {2735 err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s",2736 filt->action, filt->addr, filt->size, at, fn);2737 } else {2738 err = asprintf(&filter, "%s 0x%"PRIx64"%s%s",2739 filt->action, filt->addr, at, fn);2740 }2741 2742 return err < 0 ? NULL : filter;2743}2744 2745static int parse_addr_filter(struct evsel *evsel, const char *filter,2746 int max_nr)2747{2748 struct addr_filters filts;2749 struct addr_filter *filt;2750 int err;2751 2752 addr_filters__init(&filts);2753 2754 err = addr_filters__parse_bare_filter(&filts, filter);2755 if (err)2756 goto out_exit;2757 2758 if (filts.cnt > max_nr) {2759 pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n",2760 filts.cnt, max_nr);2761 err = -EINVAL;2762 goto out_exit;2763 }2764 2765 list_for_each_entry(filt, &filts.head, list) {2766 char *new_filter;2767 2768 err = addr_filter__resolve_syms(filt);2769 if (err)2770 goto out_exit;2771 2772 new_filter = addr_filter__to_str(filt);2773 if (!new_filter) {2774 err = -ENOMEM;2775 goto out_exit;2776 }2777 2778 if (evsel__append_addr_filter(evsel, new_filter)) {2779 err = -ENOMEM;2780 goto out_exit;2781 }2782 }2783 2784out_exit:2785 addr_filters__exit(&filts);2786 2787 if (err) {2788 pr_err("Failed to parse address filter: '%s'\n", filter);2789 pr_err("Filter format is: filter|start|stop|tracestop <start symbol or address> [/ <end symbol or size>] [@<file name>]\n");2790 pr_err("Where multiple filters are separated by space or comma.\n");2791 }2792 2793 return err;2794}2795 2796static int evsel__nr_addr_filter(struct evsel *evsel)2797{2798 struct perf_pmu *pmu = evsel__find_pmu(evsel);2799 int nr_addr_filters = 0;2800 2801 if (!pmu)2802 return 0;2803 2804 perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters);2805 2806 return nr_addr_filters;2807}2808 2809int auxtrace_parse_filters(struct evlist *evlist)2810{2811 struct evsel *evsel;2812 char *filter;2813 int err, max_nr;2814 2815 evlist__for_each_entry(evlist, evsel) {2816 filter = evsel->filter;2817 max_nr = evsel__nr_addr_filter(evsel);2818 if (!filter || !max_nr)2819 continue;2820 evsel->filter = NULL;2821 err = parse_addr_filter(evsel, filter, max_nr);2822 free(filter);2823 if (err)2824 return err;2825 pr_debug("Address filter: %s\n", evsel->filter);2826 }2827 2828 return 0;2829}2830 2831int auxtrace__process_event(struct perf_session *session, union perf_event *event,2832 struct perf_sample *sample, const struct perf_tool *tool)2833{2834 if (!session->auxtrace)2835 return 0;2836 2837 return session->auxtrace->process_event(session, event, sample, tool);2838}2839 2840void auxtrace__dump_auxtrace_sample(struct perf_session *session,2841 struct perf_sample *sample)2842{2843 if (!session->auxtrace || !session->auxtrace->dump_auxtrace_sample ||2844 auxtrace__dont_decode(session))2845 return;2846 2847 session->auxtrace->dump_auxtrace_sample(session, sample);2848}2849 2850int auxtrace__flush_events(struct perf_session *session, const struct perf_tool *tool)2851{2852 if (!session->auxtrace)2853 return 0;2854 2855 return session->auxtrace->flush_events(session, tool);2856}2857 2858void auxtrace__free_events(struct perf_session *session)2859{2860 if (!session->auxtrace)2861 return;2862 2863 return session->auxtrace->free_events(session);2864}2865 2866void auxtrace__free(struct perf_session *session)2867{2868 if (!session->auxtrace)2869 return;2870 2871 return session->auxtrace->free(session);2872}2873 2874bool auxtrace__evsel_is_auxtrace(struct perf_session *session,2875 struct evsel *evsel)2876{2877 if (!session->auxtrace || !session->auxtrace->evsel_is_auxtrace)2878 return false;2879 2880 return session->auxtrace->evsel_is_auxtrace(session, evsel);2881}2882