2065 lines · c
1// SPDX-License-Identifier: GPL-2.02#include "builtin.h"3 4#include "util/dso.h"5#include "util/evlist.h"6#include "util/evsel.h"7#include "util/config.h"8#include "util/map.h"9#include "util/symbol.h"10#include "util/thread.h"11#include "util/header.h"12#include "util/session.h"13#include "util/tool.h"14#include "util/callchain.h"15#include "util/time-utils.h"16#include <linux/err.h>17 18#include <subcmd/pager.h>19#include <subcmd/parse-options.h>20#include "util/trace-event.h"21#include "util/data.h"22#include "util/cpumap.h"23 24#include "util/debug.h"25#include "util/string2.h"26#include "util/util.h"27 28#include <linux/kernel.h>29#include <linux/numa.h>30#include <linux/rbtree.h>31#include <linux/string.h>32#include <linux/zalloc.h>33#include <errno.h>34#include <inttypes.h>35#include <locale.h>36#include <regex.h>37 38#include <linux/ctype.h>39#include <traceevent/event-parse.h>40 41static int kmem_slab;42static int kmem_page;43 44static long kmem_page_size;45static enum {46 KMEM_SLAB,47 KMEM_PAGE,48} kmem_default = KMEM_SLAB; /* for backward compatibility */49 50struct alloc_stat;51typedef int (*sort_fn_t)(void *, void *);52 53static int alloc_flag;54static int caller_flag;55 56static int alloc_lines = -1;57static int caller_lines = -1;58 59static bool raw_ip;60 61struct alloc_stat {62 u64 call_site;63 u64 ptr;64 u64 bytes_req;65 u64 bytes_alloc;66 u64 last_alloc;67 u32 hit;68 u32 pingpong;69 70 short alloc_cpu;71 72 struct rb_node node;73};74 75static struct rb_root root_alloc_stat;76static struct rb_root root_alloc_sorted;77static struct rb_root root_caller_stat;78static struct rb_root root_caller_sorted;79 80static unsigned long total_requested, total_allocated, total_freed;81static unsigned long nr_allocs, nr_cross_allocs;82 83/* filters for controlling start and stop of time of analysis */84static struct perf_time_interval ptime;85const char *time_str;86 87static int insert_alloc_stat(unsigned long call_site, unsigned long ptr,88 int bytes_req, int bytes_alloc, int cpu)89{90 struct rb_node **node = &root_alloc_stat.rb_node;91 struct rb_node *parent = NULL;92 struct alloc_stat *data = NULL;93 94 while (*node) {95 parent = *node;96 data = rb_entry(*node, struct alloc_stat, node);97 98 if (ptr > data->ptr)99 node = &(*node)->rb_right;100 else if (ptr < data->ptr)101 node = &(*node)->rb_left;102 else103 break;104 }105 106 if (data && data->ptr == ptr) {107 data->hit++;108 data->bytes_req += bytes_req;109 data->bytes_alloc += bytes_alloc;110 } else {111 data = malloc(sizeof(*data));112 if (!data) {113 pr_err("%s: malloc failed\n", __func__);114 return -1;115 }116 data->ptr = ptr;117 data->pingpong = 0;118 data->hit = 1;119 data->bytes_req = bytes_req;120 data->bytes_alloc = bytes_alloc;121 122 rb_link_node(&data->node, parent, node);123 rb_insert_color(&data->node, &root_alloc_stat);124 }125 data->call_site = call_site;126 data->alloc_cpu = cpu;127 data->last_alloc = bytes_alloc;128 129 return 0;130}131 132static int insert_caller_stat(unsigned long call_site,133 int bytes_req, int bytes_alloc)134{135 struct rb_node **node = &root_caller_stat.rb_node;136 struct rb_node *parent = NULL;137 struct alloc_stat *data = NULL;138 139 while (*node) {140 parent = *node;141 data = rb_entry(*node, struct alloc_stat, node);142 143 if (call_site > data->call_site)144 node = &(*node)->rb_right;145 else if (call_site < data->call_site)146 node = &(*node)->rb_left;147 else148 break;149 }150 151 if (data && data->call_site == call_site) {152 data->hit++;153 data->bytes_req += bytes_req;154 data->bytes_alloc += bytes_alloc;155 } else {156 data = malloc(sizeof(*data));157 if (!data) {158 pr_err("%s: malloc failed\n", __func__);159 return -1;160 }161 data->call_site = call_site;162 data->pingpong = 0;163 data->hit = 1;164 data->bytes_req = bytes_req;165 data->bytes_alloc = bytes_alloc;166 167 rb_link_node(&data->node, parent, node);168 rb_insert_color(&data->node, &root_caller_stat);169 }170 171 return 0;172}173 174static int evsel__process_alloc_event(struct evsel *evsel, struct perf_sample *sample)175{176 unsigned long ptr = evsel__intval(evsel, sample, "ptr"),177 call_site = evsel__intval(evsel, sample, "call_site");178 int bytes_req = evsel__intval(evsel, sample, "bytes_req"),179 bytes_alloc = evsel__intval(evsel, sample, "bytes_alloc");180 181 if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, sample->cpu) ||182 insert_caller_stat(call_site, bytes_req, bytes_alloc))183 return -1;184 185 total_requested += bytes_req;186 total_allocated += bytes_alloc;187 188 nr_allocs++;189 190 /*191 * Commit 11e9734bcb6a ("mm/slab_common: unify NUMA and UMA192 * version of tracepoints") adds the field "node" into the193 * tracepoints 'kmalloc' and 'kmem_cache_alloc'.194 *195 * The legacy tracepoints 'kmalloc_node' and 'kmem_cache_alloc_node'196 * also contain the field "node".197 *198 * If the tracepoint contains the field "node" the tool stats the199 * cross allocation.200 */201 if (evsel__field(evsel, "node")) {202 int node1, node2;203 204 node1 = cpu__get_node((struct perf_cpu){.cpu = sample->cpu});205 node2 = evsel__intval(evsel, sample, "node");206 207 /*208 * If the field "node" is NUMA_NO_NODE (-1), we don't take it209 * as a cross allocation.210 */211 if ((node2 != NUMA_NO_NODE) && (node1 != node2))212 nr_cross_allocs++;213 }214 215 return 0;216}217 218static int ptr_cmp(void *, void *);219static int slab_callsite_cmp(void *, void *);220 221static struct alloc_stat *search_alloc_stat(unsigned long ptr,222 unsigned long call_site,223 struct rb_root *root,224 sort_fn_t sort_fn)225{226 struct rb_node *node = root->rb_node;227 struct alloc_stat key = { .ptr = ptr, .call_site = call_site };228 229 while (node) {230 struct alloc_stat *data;231 int cmp;232 233 data = rb_entry(node, struct alloc_stat, node);234 235 cmp = sort_fn(&key, data);236 if (cmp < 0)237 node = node->rb_left;238 else if (cmp > 0)239 node = node->rb_right;240 else241 return data;242 }243 return NULL;244}245 246static int evsel__process_free_event(struct evsel *evsel, struct perf_sample *sample)247{248 unsigned long ptr = evsel__intval(evsel, sample, "ptr");249 struct alloc_stat *s_alloc, *s_caller;250 251 s_alloc = search_alloc_stat(ptr, 0, &root_alloc_stat, ptr_cmp);252 if (!s_alloc)253 return 0;254 255 total_freed += s_alloc->last_alloc;256 257 if ((short)sample->cpu != s_alloc->alloc_cpu) {258 s_alloc->pingpong++;259 260 s_caller = search_alloc_stat(0, s_alloc->call_site,261 &root_caller_stat,262 slab_callsite_cmp);263 if (!s_caller)264 return -1;265 s_caller->pingpong++;266 }267 s_alloc->alloc_cpu = -1;268 269 return 0;270}271 272static u64 total_page_alloc_bytes;273static u64 total_page_free_bytes;274static u64 total_page_nomatch_bytes;275static u64 total_page_fail_bytes;276static unsigned long nr_page_allocs;277static unsigned long nr_page_frees;278static unsigned long nr_page_fails;279static unsigned long nr_page_nomatch;280 281static bool use_pfn;282static bool live_page;283static struct perf_session *kmem_session;284 285#define MAX_MIGRATE_TYPES 6286#define MAX_PAGE_ORDER 11287 288static int order_stats[MAX_PAGE_ORDER][MAX_MIGRATE_TYPES];289 290struct page_stat {291 struct rb_node node;292 u64 page;293 u64 callsite;294 int order;295 unsigned gfp_flags;296 unsigned migrate_type;297 u64 alloc_bytes;298 u64 free_bytes;299 int nr_alloc;300 int nr_free;301};302 303static struct rb_root page_live_tree;304static struct rb_root page_alloc_tree;305static struct rb_root page_alloc_sorted;306static struct rb_root page_caller_tree;307static struct rb_root page_caller_sorted;308 309struct alloc_func {310 u64 start;311 u64 end;312 char *name;313};314 315static int nr_alloc_funcs;316static struct alloc_func *alloc_func_list;317 318static int funcmp(const void *a, const void *b)319{320 const struct alloc_func *fa = a;321 const struct alloc_func *fb = b;322 323 if (fa->start > fb->start)324 return 1;325 else326 return -1;327}328 329static int callcmp(const void *a, const void *b)330{331 const struct alloc_func *fa = a;332 const struct alloc_func *fb = b;333 334 if (fb->start <= fa->start && fa->end < fb->end)335 return 0;336 337 if (fa->start > fb->start)338 return 1;339 else340 return -1;341}342 343static int build_alloc_func_list(void)344{345 int ret;346 struct map *kernel_map;347 struct symbol *sym;348 struct rb_node *node;349 struct alloc_func *func;350 struct machine *machine = &kmem_session->machines.host;351 regex_t alloc_func_regex;352 static const char pattern[] = "^_?_?(alloc|get_free|get_zeroed)_pages?";353 354 ret = regcomp(&alloc_func_regex, pattern, REG_EXTENDED);355 if (ret) {356 char err[BUFSIZ];357 358 regerror(ret, &alloc_func_regex, err, sizeof(err));359 pr_err("Invalid regex: %s\n%s", pattern, err);360 return -EINVAL;361 }362 363 kernel_map = machine__kernel_map(machine);364 if (map__load(kernel_map) < 0) {365 pr_err("cannot load kernel map\n");366 return -ENOENT;367 }368 369 map__for_each_symbol(kernel_map, sym, node) {370 if (regexec(&alloc_func_regex, sym->name, 0, NULL, 0))371 continue;372 373 func = realloc(alloc_func_list,374 (nr_alloc_funcs + 1) * sizeof(*func));375 if (func == NULL)376 return -ENOMEM;377 378 pr_debug("alloc func: %s\n", sym->name);379 func[nr_alloc_funcs].start = sym->start;380 func[nr_alloc_funcs].end = sym->end;381 func[nr_alloc_funcs].name = sym->name;382 383 alloc_func_list = func;384 nr_alloc_funcs++;385 }386 387 qsort(alloc_func_list, nr_alloc_funcs, sizeof(*func), funcmp);388 389 regfree(&alloc_func_regex);390 return 0;391}392 393/*394 * Find first non-memory allocation function from callchain.395 * The allocation functions are in the 'alloc_func_list'.396 */397static u64 find_callsite(struct evsel *evsel, struct perf_sample *sample)398{399 struct addr_location al;400 struct machine *machine = &kmem_session->machines.host;401 struct callchain_cursor_node *node;402 struct callchain_cursor *cursor;403 u64 result = sample->ip;404 405 addr_location__init(&al);406 if (alloc_func_list == NULL) {407 if (build_alloc_func_list() < 0)408 goto out;409 }410 411 al.thread = machine__findnew_thread(machine, sample->pid, sample->tid);412 413 cursor = get_tls_callchain_cursor();414 if (cursor == NULL)415 goto out;416 417 sample__resolve_callchain(sample, cursor, NULL, evsel, &al, 16);418 419 callchain_cursor_commit(cursor);420 while (true) {421 struct alloc_func key, *caller;422 u64 addr;423 424 node = callchain_cursor_current(cursor);425 if (node == NULL)426 break;427 428 key.start = key.end = node->ip;429 caller = bsearch(&key, alloc_func_list, nr_alloc_funcs,430 sizeof(key), callcmp);431 if (!caller) {432 /* found */433 if (node->ms.map)434 addr = map__dso_unmap_ip(node->ms.map, node->ip);435 else436 addr = node->ip;437 438 result = addr;439 goto out;440 } else441 pr_debug3("skipping alloc function: %s\n", caller->name);442 443 callchain_cursor_advance(cursor);444 }445 446 pr_debug2("unknown callsite: %"PRIx64 "\n", sample->ip);447out:448 addr_location__exit(&al);449 return result;450}451 452struct sort_dimension {453 const char name[20];454 sort_fn_t cmp;455 struct list_head list;456};457 458static LIST_HEAD(page_alloc_sort_input);459static LIST_HEAD(page_caller_sort_input);460 461static struct page_stat *462__page_stat__findnew_page(struct page_stat *pstat, bool create)463{464 struct rb_node **node = &page_live_tree.rb_node;465 struct rb_node *parent = NULL;466 struct page_stat *data;467 468 while (*node) {469 s64 cmp;470 471 parent = *node;472 data = rb_entry(*node, struct page_stat, node);473 474 cmp = data->page - pstat->page;475 if (cmp < 0)476 node = &parent->rb_left;477 else if (cmp > 0)478 node = &parent->rb_right;479 else480 return data;481 }482 483 if (!create)484 return NULL;485 486 data = zalloc(sizeof(*data));487 if (data != NULL) {488 data->page = pstat->page;489 data->order = pstat->order;490 data->gfp_flags = pstat->gfp_flags;491 data->migrate_type = pstat->migrate_type;492 493 rb_link_node(&data->node, parent, node);494 rb_insert_color(&data->node, &page_live_tree);495 }496 497 return data;498}499 500static struct page_stat *page_stat__find_page(struct page_stat *pstat)501{502 return __page_stat__findnew_page(pstat, false);503}504 505static struct page_stat *page_stat__findnew_page(struct page_stat *pstat)506{507 return __page_stat__findnew_page(pstat, true);508}509 510static struct page_stat *511__page_stat__findnew_alloc(struct page_stat *pstat, bool create)512{513 struct rb_node **node = &page_alloc_tree.rb_node;514 struct rb_node *parent = NULL;515 struct page_stat *data;516 struct sort_dimension *sort;517 518 while (*node) {519 int cmp = 0;520 521 parent = *node;522 data = rb_entry(*node, struct page_stat, node);523 524 list_for_each_entry(sort, &page_alloc_sort_input, list) {525 cmp = sort->cmp(pstat, data);526 if (cmp)527 break;528 }529 530 if (cmp < 0)531 node = &parent->rb_left;532 else if (cmp > 0)533 node = &parent->rb_right;534 else535 return data;536 }537 538 if (!create)539 return NULL;540 541 data = zalloc(sizeof(*data));542 if (data != NULL) {543 data->page = pstat->page;544 data->order = pstat->order;545 data->gfp_flags = pstat->gfp_flags;546 data->migrate_type = pstat->migrate_type;547 548 rb_link_node(&data->node, parent, node);549 rb_insert_color(&data->node, &page_alloc_tree);550 }551 552 return data;553}554 555static struct page_stat *page_stat__find_alloc(struct page_stat *pstat)556{557 return __page_stat__findnew_alloc(pstat, false);558}559 560static struct page_stat *page_stat__findnew_alloc(struct page_stat *pstat)561{562 return __page_stat__findnew_alloc(pstat, true);563}564 565static struct page_stat *566__page_stat__findnew_caller(struct page_stat *pstat, bool create)567{568 struct rb_node **node = &page_caller_tree.rb_node;569 struct rb_node *parent = NULL;570 struct page_stat *data;571 struct sort_dimension *sort;572 573 while (*node) {574 int cmp = 0;575 576 parent = *node;577 data = rb_entry(*node, struct page_stat, node);578 579 list_for_each_entry(sort, &page_caller_sort_input, list) {580 cmp = sort->cmp(pstat, data);581 if (cmp)582 break;583 }584 585 if (cmp < 0)586 node = &parent->rb_left;587 else if (cmp > 0)588 node = &parent->rb_right;589 else590 return data;591 }592 593 if (!create)594 return NULL;595 596 data = zalloc(sizeof(*data));597 if (data != NULL) {598 data->callsite = pstat->callsite;599 data->order = pstat->order;600 data->gfp_flags = pstat->gfp_flags;601 data->migrate_type = pstat->migrate_type;602 603 rb_link_node(&data->node, parent, node);604 rb_insert_color(&data->node, &page_caller_tree);605 }606 607 return data;608}609 610static struct page_stat *page_stat__find_caller(struct page_stat *pstat)611{612 return __page_stat__findnew_caller(pstat, false);613}614 615static struct page_stat *page_stat__findnew_caller(struct page_stat *pstat)616{617 return __page_stat__findnew_caller(pstat, true);618}619 620static bool valid_page(u64 pfn_or_page)621{622 if (use_pfn && pfn_or_page == -1UL)623 return false;624 if (!use_pfn && pfn_or_page == 0)625 return false;626 return true;627}628 629struct gfp_flag {630 unsigned int flags;631 char *compact_str;632 char *human_readable;633};634 635static struct gfp_flag *gfps;636static int nr_gfps;637 638static int gfpcmp(const void *a, const void *b)639{640 const struct gfp_flag *fa = a;641 const struct gfp_flag *fb = b;642 643 return fa->flags - fb->flags;644}645 646/* see include/trace/events/mmflags.h */647static const struct {648 const char *original;649 const char *compact;650} gfp_compact_table[] = {651 { "GFP_TRANSHUGE", "THP" },652 { "GFP_TRANSHUGE_LIGHT", "THL" },653 { "GFP_HIGHUSER_MOVABLE", "HUM" },654 { "GFP_HIGHUSER", "HU" },655 { "GFP_USER", "U" },656 { "GFP_KERNEL_ACCOUNT", "KAC" },657 { "GFP_KERNEL", "K" },658 { "GFP_NOFS", "NF" },659 { "GFP_ATOMIC", "A" },660 { "GFP_NOIO", "NI" },661 { "GFP_NOWAIT", "NW" },662 { "GFP_DMA", "D" },663 { "__GFP_HIGHMEM", "HM" },664 { "GFP_DMA32", "D32" },665 { "__GFP_HIGH", "H" },666 { "__GFP_IO", "I" },667 { "__GFP_FS", "F" },668 { "__GFP_NOWARN", "NWR" },669 { "__GFP_RETRY_MAYFAIL", "R" },670 { "__GFP_NOFAIL", "NF" },671 { "__GFP_NORETRY", "NR" },672 { "__GFP_COMP", "C" },673 { "__GFP_ZERO", "Z" },674 { "__GFP_NOMEMALLOC", "NMA" },675 { "__GFP_MEMALLOC", "MA" },676 { "__GFP_HARDWALL", "HW" },677 { "__GFP_THISNODE", "TN" },678 { "__GFP_RECLAIMABLE", "RC" },679 { "__GFP_MOVABLE", "M" },680 { "__GFP_ACCOUNT", "AC" },681 { "__GFP_WRITE", "WR" },682 { "__GFP_RECLAIM", "R" },683 { "__GFP_DIRECT_RECLAIM", "DR" },684 { "__GFP_KSWAPD_RECLAIM", "KR" },685};686 687static size_t max_gfp_len;688 689static char *compact_gfp_flags(char *gfp_flags)690{691 char *orig_flags = strdup(gfp_flags);692 char *new_flags = NULL;693 char *str, *pos = NULL;694 size_t len = 0;695 696 if (orig_flags == NULL)697 return NULL;698 699 str = strtok_r(orig_flags, "|", &pos);700 while (str) {701 size_t i;702 char *new;703 const char *cpt;704 705 for (i = 0; i < ARRAY_SIZE(gfp_compact_table); i++) {706 if (strcmp(gfp_compact_table[i].original, str))707 continue;708 709 cpt = gfp_compact_table[i].compact;710 new = realloc(new_flags, len + strlen(cpt) + 2);711 if (new == NULL) {712 free(new_flags);713 free(orig_flags);714 return NULL;715 }716 717 new_flags = new;718 719 if (!len) {720 strcpy(new_flags, cpt);721 } else {722 strcat(new_flags, "|");723 strcat(new_flags, cpt);724 len++;725 }726 727 len += strlen(cpt);728 }729 730 str = strtok_r(NULL, "|", &pos);731 }732 733 if (max_gfp_len < len)734 max_gfp_len = len;735 736 free(orig_flags);737 return new_flags;738}739 740static char *compact_gfp_string(unsigned long gfp_flags)741{742 struct gfp_flag key = {743 .flags = gfp_flags,744 };745 struct gfp_flag *gfp;746 747 gfp = bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp);748 if (gfp)749 return gfp->compact_str;750 751 return NULL;752}753 754static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample,755 unsigned int gfp_flags)756{757 struct tep_record record = {758 .cpu = sample->cpu,759 .data = sample->raw_data,760 .size = sample->raw_size,761 };762 struct trace_seq seq;763 char *str, *pos = NULL;764 765 if (nr_gfps) {766 struct gfp_flag key = {767 .flags = gfp_flags,768 };769 770 if (bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp))771 return 0;772 }773 774 trace_seq_init(&seq);775 tep_print_event(evsel->tp_format->tep,776 &seq, &record, "%s", TEP_PRINT_INFO);777 778 str = strtok_r(seq.buffer, " ", &pos);779 while (str) {780 if (!strncmp(str, "gfp_flags=", 10)) {781 struct gfp_flag *new;782 783 new = realloc(gfps, (nr_gfps + 1) * sizeof(*gfps));784 if (new == NULL)785 return -ENOMEM;786 787 gfps = new;788 new += nr_gfps++;789 790 new->flags = gfp_flags;791 new->human_readable = strdup(str + 10);792 new->compact_str = compact_gfp_flags(str + 10);793 if (!new->human_readable || !new->compact_str)794 return -ENOMEM;795 796 qsort(gfps, nr_gfps, sizeof(*gfps), gfpcmp);797 }798 799 str = strtok_r(NULL, " ", &pos);800 }801 802 trace_seq_destroy(&seq);803 return 0;804}805 806static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_sample *sample)807{808 u64 page;809 unsigned int order = evsel__intval(evsel, sample, "order");810 unsigned int gfp_flags = evsel__intval(evsel, sample, "gfp_flags");811 unsigned int migrate_type = evsel__intval(evsel, sample,812 "migratetype");813 u64 bytes = kmem_page_size << order;814 u64 callsite;815 struct page_stat *pstat;816 struct page_stat this = {817 .order = order,818 .gfp_flags = gfp_flags,819 .migrate_type = migrate_type,820 };821 822 if (use_pfn)823 page = evsel__intval(evsel, sample, "pfn");824 else825 page = evsel__intval(evsel, sample, "page");826 827 nr_page_allocs++;828 total_page_alloc_bytes += bytes;829 830 if (!valid_page(page)) {831 nr_page_fails++;832 total_page_fail_bytes += bytes;833 834 return 0;835 }836 837 if (parse_gfp_flags(evsel, sample, gfp_flags) < 0)838 return -1;839 840 callsite = find_callsite(evsel, sample);841 842 /*843 * This is to find the current page (with correct gfp flags and844 * migrate type) at free event.845 */846 this.page = page;847 pstat = page_stat__findnew_page(&this);848 if (pstat == NULL)849 return -ENOMEM;850 851 pstat->nr_alloc++;852 pstat->alloc_bytes += bytes;853 pstat->callsite = callsite;854 855 if (!live_page) {856 pstat = page_stat__findnew_alloc(&this);857 if (pstat == NULL)858 return -ENOMEM;859 860 pstat->nr_alloc++;861 pstat->alloc_bytes += bytes;862 pstat->callsite = callsite;863 }864 865 this.callsite = callsite;866 pstat = page_stat__findnew_caller(&this);867 if (pstat == NULL)868 return -ENOMEM;869 870 pstat->nr_alloc++;871 pstat->alloc_bytes += bytes;872 873 order_stats[order][migrate_type]++;874 875 return 0;876}877 878static int evsel__process_page_free_event(struct evsel *evsel, struct perf_sample *sample)879{880 u64 page;881 unsigned int order = evsel__intval(evsel, sample, "order");882 u64 bytes = kmem_page_size << order;883 struct page_stat *pstat;884 struct page_stat this = {885 .order = order,886 };887 888 if (use_pfn)889 page = evsel__intval(evsel, sample, "pfn");890 else891 page = evsel__intval(evsel, sample, "page");892 893 nr_page_frees++;894 total_page_free_bytes += bytes;895 896 this.page = page;897 pstat = page_stat__find_page(&this);898 if (pstat == NULL) {899 pr_debug2("missing free at page %"PRIx64" (order: %d)\n",900 page, order);901 902 nr_page_nomatch++;903 total_page_nomatch_bytes += bytes;904 905 return 0;906 }907 908 this.gfp_flags = pstat->gfp_flags;909 this.migrate_type = pstat->migrate_type;910 this.callsite = pstat->callsite;911 912 rb_erase(&pstat->node, &page_live_tree);913 free(pstat);914 915 if (live_page) {916 order_stats[this.order][this.migrate_type]--;917 } else {918 pstat = page_stat__find_alloc(&this);919 if (pstat == NULL)920 return -ENOMEM;921 922 pstat->nr_free++;923 pstat->free_bytes += bytes;924 }925 926 pstat = page_stat__find_caller(&this);927 if (pstat == NULL)928 return -ENOENT;929 930 pstat->nr_free++;931 pstat->free_bytes += bytes;932 933 if (live_page) {934 pstat->nr_alloc--;935 pstat->alloc_bytes -= bytes;936 937 if (pstat->nr_alloc == 0) {938 rb_erase(&pstat->node, &page_caller_tree);939 free(pstat);940 }941 }942 943 return 0;944}945 946static bool perf_kmem__skip_sample(struct perf_sample *sample)947{948 /* skip sample based on time? */949 if (perf_time__skip_sample(&ptime, sample->time))950 return true;951 952 return false;953}954 955typedef int (*tracepoint_handler)(struct evsel *evsel,956 struct perf_sample *sample);957 958static int process_sample_event(const struct perf_tool *tool __maybe_unused,959 union perf_event *event,960 struct perf_sample *sample,961 struct evsel *evsel,962 struct machine *machine)963{964 int err = 0;965 struct thread *thread = machine__findnew_thread(machine, sample->pid,966 sample->tid);967 968 if (thread == NULL) {969 pr_debug("problem processing %d event, skipping it.\n",970 event->header.type);971 return -1;972 }973 974 if (perf_kmem__skip_sample(sample))975 return 0;976 977 dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread__tid(thread));978 979 if (evsel->handler != NULL) {980 tracepoint_handler f = evsel->handler;981 err = f(evsel, sample);982 }983 984 thread__put(thread);985 986 return err;987}988 989static double fragmentation(unsigned long n_req, unsigned long n_alloc)990{991 if (n_alloc == 0)992 return 0.0;993 else994 return 100.0 - (100.0 * n_req / n_alloc);995}996 997static void __print_slab_result(struct rb_root *root,998 struct perf_session *session,999 int n_lines, int is_caller)1000{1001 struct rb_node *next;1002 struct machine *machine = &session->machines.host;1003 1004 printf("%.105s\n", graph_dotted_line);1005 printf(" %-34s |", is_caller ? "Callsite": "Alloc Ptr");1006 printf(" Total_alloc/Per | Total_req/Per | Hit | Ping-pong | Frag\n");1007 printf("%.105s\n", graph_dotted_line);1008 1009 next = rb_first(root);1010 1011 while (next && n_lines--) {1012 struct alloc_stat *data = rb_entry(next, struct alloc_stat,1013 node);1014 struct symbol *sym = NULL;1015 struct map *map;1016 char buf[BUFSIZ];1017 u64 addr;1018 1019 if (is_caller) {1020 addr = data->call_site;1021 if (!raw_ip)1022 sym = machine__find_kernel_symbol(machine, addr, &map);1023 } else1024 addr = data->ptr;1025 1026 if (sym != NULL)1027 snprintf(buf, sizeof(buf), "%s+%" PRIx64 "", sym->name,1028 addr - map__unmap_ip(map, sym->start));1029 else1030 snprintf(buf, sizeof(buf), "%#" PRIx64 "", addr);1031 printf(" %-34s |", buf);1032 1033 printf(" %9llu/%-5lu | %9llu/%-5lu | %8lu | %9lu | %6.3f%%\n",1034 (unsigned long long)data->bytes_alloc,1035 (unsigned long)data->bytes_alloc / data->hit,1036 (unsigned long long)data->bytes_req,1037 (unsigned long)data->bytes_req / data->hit,1038 (unsigned long)data->hit,1039 (unsigned long)data->pingpong,1040 fragmentation(data->bytes_req, data->bytes_alloc));1041 1042 next = rb_next(next);1043 }1044 1045 if (n_lines == -1)1046 printf(" ... | ... | ... | ... | ... | ... \n");1047 1048 printf("%.105s\n", graph_dotted_line);1049}1050 1051static const char * const migrate_type_str[] = {1052 "UNMOVABL",1053 "RECLAIM",1054 "MOVABLE",1055 "RESERVED",1056 "CMA/ISLT",1057 "UNKNOWN",1058};1059 1060static void __print_page_alloc_result(struct perf_session *session, int n_lines)1061{1062 struct rb_node *next = rb_first(&page_alloc_sorted);1063 struct machine *machine = &session->machines.host;1064 const char *format;1065 int gfp_len = max(strlen("GFP flags"), max_gfp_len);1066 1067 printf("\n%.105s\n", graph_dotted_line);1068 printf(" %-16s | %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",1069 use_pfn ? "PFN" : "Page", live_page ? "Live" : "Total",1070 gfp_len, "GFP flags");1071 printf("%.105s\n", graph_dotted_line);1072 1073 if (use_pfn)1074 format = " %16llu | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";1075 else1076 format = " %016llx | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";1077 1078 while (next && n_lines--) {1079 struct page_stat *data;1080 struct symbol *sym;1081 struct map *map;1082 char buf[32];1083 char *caller = buf;1084 1085 data = rb_entry(next, struct page_stat, node);1086 sym = machine__find_kernel_symbol(machine, data->callsite, &map);1087 if (sym)1088 caller = sym->name;1089 else1090 scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);1091 1092 printf(format, (unsigned long long)data->page,1093 (unsigned long long)data->alloc_bytes / 1024,1094 data->nr_alloc, data->order,1095 migrate_type_str[data->migrate_type],1096 gfp_len, compact_gfp_string(data->gfp_flags), caller);1097 1098 next = rb_next(next);1099 }1100 1101 if (n_lines == -1) {1102 printf(" ... | ... | ... | ... | ... | %-*s | ...\n",1103 gfp_len, "...");1104 }1105 1106 printf("%.105s\n", graph_dotted_line);1107}1108 1109static void __print_page_caller_result(struct perf_session *session, int n_lines)1110{1111 struct rb_node *next = rb_first(&page_caller_sorted);1112 struct machine *machine = &session->machines.host;1113 int gfp_len = max(strlen("GFP flags"), max_gfp_len);1114 1115 printf("\n%.105s\n", graph_dotted_line);1116 printf(" %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",1117 live_page ? "Live" : "Total", gfp_len, "GFP flags");1118 printf("%.105s\n", graph_dotted_line);1119 1120 while (next && n_lines--) {1121 struct page_stat *data;1122 struct symbol *sym;1123 struct map *map;1124 char buf[32];1125 char *caller = buf;1126 1127 data = rb_entry(next, struct page_stat, node);1128 sym = machine__find_kernel_symbol(machine, data->callsite, &map);1129 if (sym)1130 caller = sym->name;1131 else1132 scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);1133 1134 printf(" %'16llu | %'9d | %5d | %8s | %-*s | %s\n",1135 (unsigned long long)data->alloc_bytes / 1024,1136 data->nr_alloc, data->order,1137 migrate_type_str[data->migrate_type],1138 gfp_len, compact_gfp_string(data->gfp_flags), caller);1139 1140 next = rb_next(next);1141 }1142 1143 if (n_lines == -1) {1144 printf(" ... | ... | ... | ... | %-*s | ...\n",1145 gfp_len, "...");1146 }1147 1148 printf("%.105s\n", graph_dotted_line);1149}1150 1151static void print_gfp_flags(void)1152{1153 int i;1154 1155 printf("#\n");1156 printf("# GFP flags\n");1157 printf("# ---------\n");1158 for (i = 0; i < nr_gfps; i++) {1159 printf("# %08x: %*s: %s\n", gfps[i].flags,1160 (int) max_gfp_len, gfps[i].compact_str,1161 gfps[i].human_readable);1162 }1163}1164 1165static void print_slab_summary(void)1166{1167 printf("\nSUMMARY (SLAB allocator)");1168 printf("\n========================\n");1169 printf("Total bytes requested: %'lu\n", total_requested);1170 printf("Total bytes allocated: %'lu\n", total_allocated);1171 printf("Total bytes freed: %'lu\n", total_freed);1172 if (total_allocated > total_freed) {1173 printf("Net total bytes allocated: %'lu\n",1174 total_allocated - total_freed);1175 }1176 printf("Total bytes wasted on internal fragmentation: %'lu\n",1177 total_allocated - total_requested);1178 printf("Internal fragmentation: %f%%\n",1179 fragmentation(total_requested, total_allocated));1180 printf("Cross CPU allocations: %'lu/%'lu\n", nr_cross_allocs, nr_allocs);1181}1182 1183static void print_page_summary(void)1184{1185 int o, m;1186 u64 nr_alloc_freed = nr_page_frees - nr_page_nomatch;1187 u64 total_alloc_freed_bytes = total_page_free_bytes - total_page_nomatch_bytes;1188 1189 printf("\nSUMMARY (page allocator)");1190 printf("\n========================\n");1191 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation requests",1192 nr_page_allocs, total_page_alloc_bytes / 1024);1193 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free requests",1194 nr_page_frees, total_page_free_bytes / 1024);1195 printf("\n");1196 1197 printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc+freed requests",1198 nr_alloc_freed, (total_alloc_freed_bytes) / 1024);1199 printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc-only requests",1200 nr_page_allocs - nr_alloc_freed,1201 (total_page_alloc_bytes - total_alloc_freed_bytes) / 1024);1202 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free-only requests",1203 nr_page_nomatch, total_page_nomatch_bytes / 1024);1204 printf("\n");1205 1206 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation failures",1207 nr_page_fails, total_page_fail_bytes / 1024);1208 printf("\n");1209 1210 printf("%5s %12s %12s %12s %12s %12s\n", "Order", "Unmovable",1211 "Reclaimable", "Movable", "Reserved", "CMA/Isolated");1212 printf("%.5s %.12s %.12s %.12s %.12s %.12s\n", graph_dotted_line,1213 graph_dotted_line, graph_dotted_line, graph_dotted_line,1214 graph_dotted_line, graph_dotted_line);1215 1216 for (o = 0; o < MAX_PAGE_ORDER; o++) {1217 printf("%5d", o);1218 for (m = 0; m < MAX_MIGRATE_TYPES - 1; m++) {1219 if (order_stats[o][m])1220 printf(" %'12d", order_stats[o][m]);1221 else1222 printf(" %12c", '.');1223 }1224 printf("\n");1225 }1226}1227 1228static void print_slab_result(struct perf_session *session)1229{1230 if (caller_flag)1231 __print_slab_result(&root_caller_sorted, session, caller_lines, 1);1232 if (alloc_flag)1233 __print_slab_result(&root_alloc_sorted, session, alloc_lines, 0);1234 print_slab_summary();1235}1236 1237static void print_page_result(struct perf_session *session)1238{1239 if (caller_flag || alloc_flag)1240 print_gfp_flags();1241 if (caller_flag)1242 __print_page_caller_result(session, caller_lines);1243 if (alloc_flag)1244 __print_page_alloc_result(session, alloc_lines);1245 print_page_summary();1246}1247 1248static void print_result(struct perf_session *session)1249{1250 if (kmem_slab)1251 print_slab_result(session);1252 if (kmem_page)1253 print_page_result(session);1254}1255 1256static LIST_HEAD(slab_caller_sort);1257static LIST_HEAD(slab_alloc_sort);1258static LIST_HEAD(page_caller_sort);1259static LIST_HEAD(page_alloc_sort);1260 1261static void sort_slab_insert(struct rb_root *root, struct alloc_stat *data,1262 struct list_head *sort_list)1263{1264 struct rb_node **new = &(root->rb_node);1265 struct rb_node *parent = NULL;1266 struct sort_dimension *sort;1267 1268 while (*new) {1269 struct alloc_stat *this;1270 int cmp = 0;1271 1272 this = rb_entry(*new, struct alloc_stat, node);1273 parent = *new;1274 1275 list_for_each_entry(sort, sort_list, list) {1276 cmp = sort->cmp(data, this);1277 if (cmp)1278 break;1279 }1280 1281 if (cmp > 0)1282 new = &((*new)->rb_left);1283 else1284 new = &((*new)->rb_right);1285 }1286 1287 rb_link_node(&data->node, parent, new);1288 rb_insert_color(&data->node, root);1289}1290 1291static void __sort_slab_result(struct rb_root *root, struct rb_root *root_sorted,1292 struct list_head *sort_list)1293{1294 struct rb_node *node;1295 struct alloc_stat *data;1296 1297 for (;;) {1298 node = rb_first(root);1299 if (!node)1300 break;1301 1302 rb_erase(node, root);1303 data = rb_entry(node, struct alloc_stat, node);1304 sort_slab_insert(root_sorted, data, sort_list);1305 }1306}1307 1308static void sort_page_insert(struct rb_root *root, struct page_stat *data,1309 struct list_head *sort_list)1310{1311 struct rb_node **new = &root->rb_node;1312 struct rb_node *parent = NULL;1313 struct sort_dimension *sort;1314 1315 while (*new) {1316 struct page_stat *this;1317 int cmp = 0;1318 1319 this = rb_entry(*new, struct page_stat, node);1320 parent = *new;1321 1322 list_for_each_entry(sort, sort_list, list) {1323 cmp = sort->cmp(data, this);1324 if (cmp)1325 break;1326 }1327 1328 if (cmp > 0)1329 new = &parent->rb_left;1330 else1331 new = &parent->rb_right;1332 }1333 1334 rb_link_node(&data->node, parent, new);1335 rb_insert_color(&data->node, root);1336}1337 1338static void __sort_page_result(struct rb_root *root, struct rb_root *root_sorted,1339 struct list_head *sort_list)1340{1341 struct rb_node *node;1342 struct page_stat *data;1343 1344 for (;;) {1345 node = rb_first(root);1346 if (!node)1347 break;1348 1349 rb_erase(node, root);1350 data = rb_entry(node, struct page_stat, node);1351 sort_page_insert(root_sorted, data, sort_list);1352 }1353}1354 1355static void sort_result(void)1356{1357 if (kmem_slab) {1358 __sort_slab_result(&root_alloc_stat, &root_alloc_sorted,1359 &slab_alloc_sort);1360 __sort_slab_result(&root_caller_stat, &root_caller_sorted,1361 &slab_caller_sort);1362 }1363 if (kmem_page) {1364 if (live_page)1365 __sort_page_result(&page_live_tree, &page_alloc_sorted,1366 &page_alloc_sort);1367 else1368 __sort_page_result(&page_alloc_tree, &page_alloc_sorted,1369 &page_alloc_sort);1370 1371 __sort_page_result(&page_caller_tree, &page_caller_sorted,1372 &page_caller_sort);1373 }1374}1375 1376static int __cmd_kmem(struct perf_session *session)1377{1378 int err = -EINVAL;1379 struct evsel *evsel;1380 const struct evsel_str_handler kmem_tracepoints[] = {1381 /* slab allocator */1382 { "kmem:kmalloc", evsel__process_alloc_event, },1383 { "kmem:kmem_cache_alloc", evsel__process_alloc_event, },1384 { "kmem:kmalloc_node", evsel__process_alloc_event, },1385 { "kmem:kmem_cache_alloc_node", evsel__process_alloc_event, },1386 { "kmem:kfree", evsel__process_free_event, },1387 { "kmem:kmem_cache_free", evsel__process_free_event, },1388 /* page allocator */1389 { "kmem:mm_page_alloc", evsel__process_page_alloc_event, },1390 { "kmem:mm_page_free", evsel__process_page_free_event, },1391 };1392 1393 if (!perf_session__has_traces(session, "kmem record"))1394 goto out;1395 1396 if (perf_session__set_tracepoints_handlers(session, kmem_tracepoints)) {1397 pr_err("Initializing perf session tracepoint handlers failed\n");1398 goto out;1399 }1400 1401 evlist__for_each_entry(session->evlist, evsel) {1402 if (evsel__name_is(evsel, "kmem:mm_page_alloc") &&1403 evsel__field(evsel, "pfn")) {1404 use_pfn = true;1405 break;1406 }1407 }1408 1409 setup_pager();1410 err = perf_session__process_events(session);1411 if (err != 0) {1412 pr_err("error during process events: %d\n", err);1413 goto out;1414 }1415 sort_result();1416 print_result(session);1417out:1418 return err;1419}1420 1421/* slab sort keys */1422static int ptr_cmp(void *a, void *b)1423{1424 struct alloc_stat *l = a;1425 struct alloc_stat *r = b;1426 1427 if (l->ptr < r->ptr)1428 return -1;1429 else if (l->ptr > r->ptr)1430 return 1;1431 return 0;1432}1433 1434static struct sort_dimension ptr_sort_dimension = {1435 .name = "ptr",1436 .cmp = ptr_cmp,1437};1438 1439static int slab_callsite_cmp(void *a, void *b)1440{1441 struct alloc_stat *l = a;1442 struct alloc_stat *r = b;1443 1444 if (l->call_site < r->call_site)1445 return -1;1446 else if (l->call_site > r->call_site)1447 return 1;1448 return 0;1449}1450 1451static struct sort_dimension callsite_sort_dimension = {1452 .name = "callsite",1453 .cmp = slab_callsite_cmp,1454};1455 1456static int hit_cmp(void *a, void *b)1457{1458 struct alloc_stat *l = a;1459 struct alloc_stat *r = b;1460 1461 if (l->hit < r->hit)1462 return -1;1463 else if (l->hit > r->hit)1464 return 1;1465 return 0;1466}1467 1468static struct sort_dimension hit_sort_dimension = {1469 .name = "hit",1470 .cmp = hit_cmp,1471};1472 1473static int bytes_cmp(void *a, void *b)1474{1475 struct alloc_stat *l = a;1476 struct alloc_stat *r = b;1477 1478 if (l->bytes_alloc < r->bytes_alloc)1479 return -1;1480 else if (l->bytes_alloc > r->bytes_alloc)1481 return 1;1482 return 0;1483}1484 1485static struct sort_dimension bytes_sort_dimension = {1486 .name = "bytes",1487 .cmp = bytes_cmp,1488};1489 1490static int frag_cmp(void *a, void *b)1491{1492 double x, y;1493 struct alloc_stat *l = a;1494 struct alloc_stat *r = b;1495 1496 x = fragmentation(l->bytes_req, l->bytes_alloc);1497 y = fragmentation(r->bytes_req, r->bytes_alloc);1498 1499 if (x < y)1500 return -1;1501 else if (x > y)1502 return 1;1503 return 0;1504}1505 1506static struct sort_dimension frag_sort_dimension = {1507 .name = "frag",1508 .cmp = frag_cmp,1509};1510 1511static int pingpong_cmp(void *a, void *b)1512{1513 struct alloc_stat *l = a;1514 struct alloc_stat *r = b;1515 1516 if (l->pingpong < r->pingpong)1517 return -1;1518 else if (l->pingpong > r->pingpong)1519 return 1;1520 return 0;1521}1522 1523static struct sort_dimension pingpong_sort_dimension = {1524 .name = "pingpong",1525 .cmp = pingpong_cmp,1526};1527 1528/* page sort keys */1529static int page_cmp(void *a, void *b)1530{1531 struct page_stat *l = a;1532 struct page_stat *r = b;1533 1534 if (l->page < r->page)1535 return -1;1536 else if (l->page > r->page)1537 return 1;1538 return 0;1539}1540 1541static struct sort_dimension page_sort_dimension = {1542 .name = "page",1543 .cmp = page_cmp,1544};1545 1546static int page_callsite_cmp(void *a, void *b)1547{1548 struct page_stat *l = a;1549 struct page_stat *r = b;1550 1551 if (l->callsite < r->callsite)1552 return -1;1553 else if (l->callsite > r->callsite)1554 return 1;1555 return 0;1556}1557 1558static struct sort_dimension page_callsite_sort_dimension = {1559 .name = "callsite",1560 .cmp = page_callsite_cmp,1561};1562 1563static int page_hit_cmp(void *a, void *b)1564{1565 struct page_stat *l = a;1566 struct page_stat *r = b;1567 1568 if (l->nr_alloc < r->nr_alloc)1569 return -1;1570 else if (l->nr_alloc > r->nr_alloc)1571 return 1;1572 return 0;1573}1574 1575static struct sort_dimension page_hit_sort_dimension = {1576 .name = "hit",1577 .cmp = page_hit_cmp,1578};1579 1580static int page_bytes_cmp(void *a, void *b)1581{1582 struct page_stat *l = a;1583 struct page_stat *r = b;1584 1585 if (l->alloc_bytes < r->alloc_bytes)1586 return -1;1587 else if (l->alloc_bytes > r->alloc_bytes)1588 return 1;1589 return 0;1590}1591 1592static struct sort_dimension page_bytes_sort_dimension = {1593 .name = "bytes",1594 .cmp = page_bytes_cmp,1595};1596 1597static int page_order_cmp(void *a, void *b)1598{1599 struct page_stat *l = a;1600 struct page_stat *r = b;1601 1602 if (l->order < r->order)1603 return -1;1604 else if (l->order > r->order)1605 return 1;1606 return 0;1607}1608 1609static struct sort_dimension page_order_sort_dimension = {1610 .name = "order",1611 .cmp = page_order_cmp,1612};1613 1614static int migrate_type_cmp(void *a, void *b)1615{1616 struct page_stat *l = a;1617 struct page_stat *r = b;1618 1619 /* for internal use to find free'd page */1620 if (l->migrate_type == -1U)1621 return 0;1622 1623 if (l->migrate_type < r->migrate_type)1624 return -1;1625 else if (l->migrate_type > r->migrate_type)1626 return 1;1627 return 0;1628}1629 1630static struct sort_dimension migrate_type_sort_dimension = {1631 .name = "migtype",1632 .cmp = migrate_type_cmp,1633};1634 1635static int gfp_flags_cmp(void *a, void *b)1636{1637 struct page_stat *l = a;1638 struct page_stat *r = b;1639 1640 /* for internal use to find free'd page */1641 if (l->gfp_flags == -1U)1642 return 0;1643 1644 if (l->gfp_flags < r->gfp_flags)1645 return -1;1646 else if (l->gfp_flags > r->gfp_flags)1647 return 1;1648 return 0;1649}1650 1651static struct sort_dimension gfp_flags_sort_dimension = {1652 .name = "gfp",1653 .cmp = gfp_flags_cmp,1654};1655 1656static struct sort_dimension *slab_sorts[] = {1657 &ptr_sort_dimension,1658 &callsite_sort_dimension,1659 &hit_sort_dimension,1660 &bytes_sort_dimension,1661 &frag_sort_dimension,1662 &pingpong_sort_dimension,1663};1664 1665static struct sort_dimension *page_sorts[] = {1666 &page_sort_dimension,1667 &page_callsite_sort_dimension,1668 &page_hit_sort_dimension,1669 &page_bytes_sort_dimension,1670 &page_order_sort_dimension,1671 &migrate_type_sort_dimension,1672 &gfp_flags_sort_dimension,1673};1674 1675static int slab_sort_dimension__add(const char *tok, struct list_head *list)1676{1677 struct sort_dimension *sort;1678 int i;1679 1680 for (i = 0; i < (int)ARRAY_SIZE(slab_sorts); i++) {1681 if (!strcmp(slab_sorts[i]->name, tok)) {1682 sort = memdup(slab_sorts[i], sizeof(*slab_sorts[i]));1683 if (!sort) {1684 pr_err("%s: memdup failed\n", __func__);1685 return -1;1686 }1687 list_add_tail(&sort->list, list);1688 return 0;1689 }1690 }1691 1692 return -1;1693}1694 1695static int page_sort_dimension__add(const char *tok, struct list_head *list)1696{1697 struct sort_dimension *sort;1698 int i;1699 1700 for (i = 0; i < (int)ARRAY_SIZE(page_sorts); i++) {1701 if (!strcmp(page_sorts[i]->name, tok)) {1702 sort = memdup(page_sorts[i], sizeof(*page_sorts[i]));1703 if (!sort) {1704 pr_err("%s: memdup failed\n", __func__);1705 return -1;1706 }1707 list_add_tail(&sort->list, list);1708 return 0;1709 }1710 }1711 1712 return -1;1713}1714 1715static int setup_slab_sorting(struct list_head *sort_list, const char *arg)1716{1717 char *tok;1718 char *str = strdup(arg);1719 char *pos = str;1720 1721 if (!str) {1722 pr_err("%s: strdup failed\n", __func__);1723 return -1;1724 }1725 1726 while (true) {1727 tok = strsep(&pos, ",");1728 if (!tok)1729 break;1730 if (slab_sort_dimension__add(tok, sort_list) < 0) {1731 pr_err("Unknown slab --sort key: '%s'", tok);1732 free(str);1733 return -1;1734 }1735 }1736 1737 free(str);1738 return 0;1739}1740 1741static int setup_page_sorting(struct list_head *sort_list, const char *arg)1742{1743 char *tok;1744 char *str = strdup(arg);1745 char *pos = str;1746 1747 if (!str) {1748 pr_err("%s: strdup failed\n", __func__);1749 return -1;1750 }1751 1752 while (true) {1753 tok = strsep(&pos, ",");1754 if (!tok)1755 break;1756 if (page_sort_dimension__add(tok, sort_list) < 0) {1757 pr_err("Unknown page --sort key: '%s'", tok);1758 free(str);1759 return -1;1760 }1761 }1762 1763 free(str);1764 return 0;1765}1766 1767static int parse_sort_opt(const struct option *opt __maybe_unused,1768 const char *arg, int unset __maybe_unused)1769{1770 if (!arg)1771 return -1;1772 1773 if (kmem_page > kmem_slab ||1774 (kmem_page == 0 && kmem_slab == 0 && kmem_default == KMEM_PAGE)) {1775 if (caller_flag > alloc_flag)1776 return setup_page_sorting(&page_caller_sort, arg);1777 else1778 return setup_page_sorting(&page_alloc_sort, arg);1779 } else {1780 if (caller_flag > alloc_flag)1781 return setup_slab_sorting(&slab_caller_sort, arg);1782 else1783 return setup_slab_sorting(&slab_alloc_sort, arg);1784 }1785 1786 return 0;1787}1788 1789static int parse_caller_opt(const struct option *opt __maybe_unused,1790 const char *arg __maybe_unused,1791 int unset __maybe_unused)1792{1793 caller_flag = (alloc_flag + 1);1794 return 0;1795}1796 1797static int parse_alloc_opt(const struct option *opt __maybe_unused,1798 const char *arg __maybe_unused,1799 int unset __maybe_unused)1800{1801 alloc_flag = (caller_flag + 1);1802 return 0;1803}1804 1805static int parse_slab_opt(const struct option *opt __maybe_unused,1806 const char *arg __maybe_unused,1807 int unset __maybe_unused)1808{1809 kmem_slab = (kmem_page + 1);1810 return 0;1811}1812 1813static int parse_page_opt(const struct option *opt __maybe_unused,1814 const char *arg __maybe_unused,1815 int unset __maybe_unused)1816{1817 kmem_page = (kmem_slab + 1);1818 return 0;1819}1820 1821static int parse_line_opt(const struct option *opt __maybe_unused,1822 const char *arg, int unset __maybe_unused)1823{1824 int lines;1825 1826 if (!arg)1827 return -1;1828 1829 lines = strtoul(arg, NULL, 10);1830 1831 if (caller_flag > alloc_flag)1832 caller_lines = lines;1833 else1834 alloc_lines = lines;1835 1836 return 0;1837}1838 1839static bool slab_legacy_tp_is_exposed(void)1840{1841 /*1842 * The tracepoints "kmem:kmalloc_node" and1843 * "kmem:kmem_cache_alloc_node" have been removed on the latest1844 * kernel, if the tracepoint "kmem:kmalloc_node" is existed it1845 * means the tool is running on an old kernel, we need to1846 * rollback to support these legacy tracepoints.1847 */1848 return IS_ERR(trace_event__tp_format("kmem", "kmalloc_node")) ?1849 false : true;1850}1851 1852static int __cmd_record(int argc, const char **argv)1853{1854 const char * const record_args[] = {1855 "record", "-a", "-R", "-c", "1",1856 };1857 const char * const slab_events[] = {1858 "-e", "kmem:kmalloc",1859 "-e", "kmem:kfree",1860 "-e", "kmem:kmem_cache_alloc",1861 "-e", "kmem:kmem_cache_free",1862 };1863 const char * const slab_legacy_events[] = {1864 "-e", "kmem:kmalloc_node",1865 "-e", "kmem:kmem_cache_alloc_node",1866 };1867 const char * const page_events[] = {1868 "-e", "kmem:mm_page_alloc",1869 "-e", "kmem:mm_page_free",1870 };1871 unsigned int rec_argc, i, j;1872 const char **rec_argv;1873 unsigned int slab_legacy_tp_exposed = slab_legacy_tp_is_exposed();1874 1875 rec_argc = ARRAY_SIZE(record_args) + argc - 1;1876 if (kmem_slab) {1877 rec_argc += ARRAY_SIZE(slab_events);1878 if (slab_legacy_tp_exposed)1879 rec_argc += ARRAY_SIZE(slab_legacy_events);1880 }1881 if (kmem_page)1882 rec_argc += ARRAY_SIZE(page_events) + 1; /* for -g */1883 1884 rec_argv = calloc(rec_argc + 1, sizeof(char *));1885 1886 if (rec_argv == NULL)1887 return -ENOMEM;1888 1889 for (i = 0; i < ARRAY_SIZE(record_args); i++)1890 rec_argv[i] = strdup(record_args[i]);1891 1892 if (kmem_slab) {1893 for (j = 0; j < ARRAY_SIZE(slab_events); j++, i++)1894 rec_argv[i] = strdup(slab_events[j]);1895 if (slab_legacy_tp_exposed) {1896 for (j = 0; j < ARRAY_SIZE(slab_legacy_events); j++, i++)1897 rec_argv[i] = strdup(slab_legacy_events[j]);1898 }1899 }1900 if (kmem_page) {1901 rec_argv[i++] = strdup("-g");1902 1903 for (j = 0; j < ARRAY_SIZE(page_events); j++, i++)1904 rec_argv[i] = strdup(page_events[j]);1905 }1906 1907 for (j = 1; j < (unsigned int)argc; j++, i++)1908 rec_argv[i] = argv[j];1909 1910 return cmd_record(i, rec_argv);1911}1912 1913static int kmem_config(const char *var, const char *value, void *cb __maybe_unused)1914{1915 if (!strcmp(var, "kmem.default")) {1916 if (!strcmp(value, "slab"))1917 kmem_default = KMEM_SLAB;1918 else if (!strcmp(value, "page"))1919 kmem_default = KMEM_PAGE;1920 else1921 pr_err("invalid default value ('slab' or 'page' required): %s\n",1922 value);1923 return 0;1924 }1925 1926 return 0;1927}1928 1929int cmd_kmem(int argc, const char **argv)1930{1931 const char * const default_slab_sort = "frag,hit,bytes";1932 const char * const default_page_sort = "bytes,hit";1933 struct perf_data data = {1934 .mode = PERF_DATA_MODE_READ,1935 };1936 const struct option kmem_options[] = {1937 OPT_STRING('i', "input", &input_name, "file", "input file name"),1938 OPT_INCR('v', "verbose", &verbose,1939 "be more verbose (show symbol address, etc)"),1940 OPT_CALLBACK_NOOPT(0, "caller", NULL, NULL,1941 "show per-callsite statistics", parse_caller_opt),1942 OPT_CALLBACK_NOOPT(0, "alloc", NULL, NULL,1943 "show per-allocation statistics", parse_alloc_opt),1944 OPT_CALLBACK('s', "sort", NULL, "key[,key2...]",1945 "sort by keys: ptr, callsite, bytes, hit, pingpong, frag, "1946 "page, order, migtype, gfp", parse_sort_opt),1947 OPT_CALLBACK('l', "line", NULL, "num", "show n lines", parse_line_opt),1948 OPT_BOOLEAN(0, "raw-ip", &raw_ip, "show raw ip instead of symbol"),1949 OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"),1950 OPT_CALLBACK_NOOPT(0, "slab", NULL, NULL, "Analyze slab allocator",1951 parse_slab_opt),1952 OPT_CALLBACK_NOOPT(0, "page", NULL, NULL, "Analyze page allocator",1953 parse_page_opt),1954 OPT_BOOLEAN(0, "live", &live_page, "Show live page stat"),1955 OPT_STRING(0, "time", &time_str, "str",1956 "Time span of interest (start,stop)"),1957 OPT_END()1958 };1959 const char *const kmem_subcommands[] = { "record", "stat", NULL };1960 const char *kmem_usage[] = {1961 NULL,1962 NULL1963 };1964 struct perf_session *session;1965 struct perf_tool perf_kmem;1966 static const char errmsg[] = "No %s allocation events found. Have you run 'perf kmem record --%s'?\n";1967 int ret = perf_config(kmem_config, NULL);1968 1969 if (ret)1970 return ret;1971 1972 argc = parse_options_subcommand(argc, argv, kmem_options,1973 kmem_subcommands, kmem_usage,1974 PARSE_OPT_STOP_AT_NON_OPTION);1975 1976 if (!argc)1977 usage_with_options(kmem_usage, kmem_options);1978 1979 if (kmem_slab == 0 && kmem_page == 0) {1980 if (kmem_default == KMEM_SLAB)1981 kmem_slab = 1;1982 else1983 kmem_page = 1;1984 }1985 1986 if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {1987 symbol__init(NULL);1988 return __cmd_record(argc, argv);1989 }1990 1991 data.path = input_name;1992 1993 perf_tool__init(&perf_kmem, /*ordered_events=*/true);1994 perf_kmem.sample = process_sample_event;1995 perf_kmem.comm = perf_event__process_comm;1996 perf_kmem.mmap = perf_event__process_mmap;1997 perf_kmem.mmap2 = perf_event__process_mmap2;1998 perf_kmem.namespaces = perf_event__process_namespaces;1999 2000 kmem_session = session = perf_session__new(&data, &perf_kmem);2001 if (IS_ERR(session))2002 return PTR_ERR(session);2003 2004 ret = -1;2005 2006 if (kmem_slab) {2007 if (!evlist__find_tracepoint_by_name(session->evlist, "kmem:kmalloc")) {2008 pr_err(errmsg, "slab", "slab");2009 goto out_delete;2010 }2011 }2012 2013 if (kmem_page) {2014 struct evsel *evsel = evlist__find_tracepoint_by_name(session->evlist, "kmem:mm_page_alloc");2015 2016 if (evsel == NULL) {2017 pr_err(errmsg, "page", "page");2018 goto out_delete;2019 }2020 2021 kmem_page_size = tep_get_page_size(evsel->tp_format->tep);2022 symbol_conf.use_callchain = true;2023 }2024 2025 symbol__init(&session->header.env);2026 2027 if (perf_time__parse_str(&ptime, time_str) != 0) {2028 pr_err("Invalid time string\n");2029 ret = -EINVAL;2030 goto out_delete;2031 }2032 2033 if (!strcmp(argv[0], "stat")) {2034 setlocale(LC_ALL, "");2035 2036 if (cpu__setup_cpunode_map())2037 goto out_delete;2038 2039 if (list_empty(&slab_caller_sort))2040 setup_slab_sorting(&slab_caller_sort, default_slab_sort);2041 if (list_empty(&slab_alloc_sort))2042 setup_slab_sorting(&slab_alloc_sort, default_slab_sort);2043 if (list_empty(&page_caller_sort))2044 setup_page_sorting(&page_caller_sort, default_page_sort);2045 if (list_empty(&page_alloc_sort))2046 setup_page_sorting(&page_alloc_sort, default_page_sort);2047 2048 if (kmem_page) {2049 setup_page_sorting(&page_alloc_sort_input,2050 "page,order,migtype,gfp");2051 setup_page_sorting(&page_caller_sort_input,2052 "callsite,order,migtype,gfp");2053 }2054 ret = __cmd_kmem(session);2055 } else2056 usage_with_options(kmem_usage, kmem_options);2057 2058out_delete:2059 perf_session__delete(session);2060 /* free usage string allocated by parse_options_subcommand */2061 free((void *)kmem_usage[0]);2062 2063 return ret;2064}2065