486 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2014, Michael Ellerman, IBM Corp.4 */5 6#define _GNU_SOURCE /* For CPU_ZERO etc. */7 8#include <sched.h>9#include <sys/wait.h>10#include <setjmp.h>11#include <signal.h>12#include <stdio.h>13#include <stdlib.h>14#include <string.h>15#include <sys/ioctl.h>16 17#include "trace.h"18#include "ebb.h"19 20 21void (*ebb_user_func)(void);22 23void ebb_hook(void)24{25 if (ebb_user_func)26 ebb_user_func();27}28 29struct ebb_state ebb_state;30 31u64 sample_period = 0x40000000ull;32 33void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask)34{35 u64 val;36 37 /* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */38 /* 3) set MMCR0[PMAE] - docs say BESCR[PME] should do this */39 val = mfspr(SPRN_MMCR0);40 mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE);41 42 /* 4) clear BESCR[PMEO] */43 mtspr(SPRN_BESCRR, BESCR_PMEO);44 45 /* 5) set BESCR[PME] */46 mtspr(SPRN_BESCRS, BESCR_PME);47 48 /* 6) rfebb 1 - done in our caller */49}50 51void reset_ebb(void)52{53 reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC);54}55 56/* Called outside of the EBB handler to check MMCR0 is sane */57int ebb_check_mmcr0(void)58{59 u64 val;60 61 val = mfspr(SPRN_MMCR0);62 if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) {63 /* It's OK if we see FC & PMAO, but not FC by itself */64 printf("Outside of loop, only FC set 0x%llx\n", val);65 return 1;66 }67 68 return 0;69}70 71bool ebb_check_count(int pmc, u64 sample_period, int fudge)72{73 u64 count, upper, lower;74 75 count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)];76 77 lower = ebb_state.stats.ebb_count * (sample_period - fudge);78 79 if (count < lower) {80 printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n",81 pmc, count, lower, lower - count);82 return false;83 }84 85 upper = ebb_state.stats.ebb_count * (sample_period + fudge);86 87 if (count > upper) {88 printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n",89 pmc, count, upper, count - upper);90 return false;91 }92 93 printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n",94 pmc, count, lower, upper, count - lower, upper - count);95 96 return true;97}98 99void standard_ebb_callee(void)100{101 int found, i;102 u64 val;103 104 val = mfspr(SPRN_BESCR);105 if (!(val & BESCR_PMEO)) {106 ebb_state.stats.spurious++;107 goto out;108 }109 110 ebb_state.stats.ebb_count++;111 trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count);112 113 val = mfspr(SPRN_MMCR0);114 trace_log_reg(ebb_state.trace, SPRN_MMCR0, val);115 116 found = 0;117 for (i = 1; i <= 6; i++) {118 if (ebb_state.pmc_enable[PMC_INDEX(i)])119 found += count_pmc(i, sample_period);120 }121 122 if (!found)123 ebb_state.stats.no_overflow++;124 125out:126 reset_ebb();127}128 129extern void ebb_handler(void);130 131void setup_ebb_handler(void (*callee)(void))132{133 u64 entry;134 135#if defined(_CALL_ELF) && _CALL_ELF == 2136 entry = (u64)ebb_handler;137#else138 struct opd139 {140 u64 entry;141 u64 toc;142 } *opd;143 144 opd = (struct opd *)ebb_handler;145 entry = opd->entry;146#endif147 printf("EBB Handler is at %#llx\n", entry);148 149 ebb_user_func = callee;150 151 /* Ensure ebb_user_func is set before we set the handler */152 mb();153 mtspr(SPRN_EBBHR, entry);154 155 /* Make sure the handler is set before we return */156 mb();157}158 159void clear_ebb_stats(void)160{161 memset(&ebb_state.stats, 0, sizeof(ebb_state.stats));162}163 164void dump_summary_ebb_state(void)165{166 printf("ebb_state:\n" \167 " ebb_count = %d\n" \168 " spurious = %d\n" \169 " negative = %d\n" \170 " no_overflow = %d\n" \171 " pmc[1] count = 0x%llx\n" \172 " pmc[2] count = 0x%llx\n" \173 " pmc[3] count = 0x%llx\n" \174 " pmc[4] count = 0x%llx\n" \175 " pmc[5] count = 0x%llx\n" \176 " pmc[6] count = 0x%llx\n",177 ebb_state.stats.ebb_count, ebb_state.stats.spurious,178 ebb_state.stats.negative, ebb_state.stats.no_overflow,179 ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1],180 ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3],181 ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]);182}183 184static char *decode_mmcr0(u32 value)185{186 static char buf[16];187 188 buf[0] = '\0';189 190 if (value & (1 << 31))191 strcat(buf, "FC ");192 if (value & (1 << 26))193 strcat(buf, "PMAE ");194 if (value & (1 << 7))195 strcat(buf, "PMAO ");196 197 return buf;198}199 200static char *decode_bescr(u64 value)201{202 static char buf[16];203 204 buf[0] = '\0';205 206 if (value & (1ull << 63))207 strcat(buf, "GE ");208 if (value & (1ull << 32))209 strcat(buf, "PMAE ");210 if (value & 1)211 strcat(buf, "PMAO ");212 213 return buf;214}215 216void dump_ebb_hw_state(void)217{218 u64 bescr;219 u32 mmcr0;220 221 mmcr0 = mfspr(SPRN_MMCR0);222 bescr = mfspr(SPRN_BESCR);223 224 printf("HW state:\n" \225 "MMCR0 0x%016x %s\n" \226 "MMCR2 0x%016lx\n" \227 "EBBHR 0x%016lx\n" \228 "BESCR 0x%016llx %s\n" \229 "PMC1 0x%016lx\n" \230 "PMC2 0x%016lx\n" \231 "PMC3 0x%016lx\n" \232 "PMC4 0x%016lx\n" \233 "PMC5 0x%016lx\n" \234 "PMC6 0x%016lx\n" \235 "SIAR 0x%016lx\n",236 mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2),237 mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr),238 mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3),239 mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6),240 mfspr(SPRN_SIAR));241}242 243void dump_ebb_state(void)244{245 dump_summary_ebb_state();246 247 dump_ebb_hw_state();248 249 trace_buffer_print(ebb_state.trace);250}251 252int count_pmc(int pmc, uint32_t sample_period)253{254 uint32_t start_value;255 u64 val;256 257 /* 0) Read PMC */258 start_value = pmc_sample_period(sample_period);259 260 val = read_pmc(pmc);261 if (val < start_value)262 ebb_state.stats.negative++;263 else264 ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value;265 266 trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val);267 268 /* 1) Reset PMC */269 write_pmc(pmc, start_value);270 271 /* Report if we overflowed */272 return val >= COUNTER_OVERFLOW;273}274 275int ebb_event_enable(struct event *e)276{277 int rc;278 279 /* Ensure any SPR writes are ordered vs us */280 mb();281 282 rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE);283 if (rc)284 return rc;285 286 rc = event_read(e);287 288 /* Ditto */289 mb();290 291 return rc;292}293 294void ebb_freeze_pmcs(void)295{296 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);297 mb();298}299 300void ebb_unfreeze_pmcs(void)301{302 /* Unfreeze counters */303 mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC);304 mb();305}306 307void ebb_global_enable(void)308{309 /* Enable EBBs globally and PMU EBBs */310 mtspr(SPRN_BESCR, 0x8000000100000000ull);311 mb();312}313 314void ebb_global_disable(void)315{316 /* Disable EBBs & freeze counters, events are still scheduled */317 mtspr(SPRN_BESCRR, BESCR_PME);318 mb();319}320 321bool ebb_is_supported(void)322{323#ifdef PPC_FEATURE2_EBB324 /* EBB requires at least POWER8 */325 return have_hwcap2(PPC_FEATURE2_EBB);326#else327 return false;328#endif329}330 331void event_ebb_init(struct event *e)332{333 e->attr.config |= (1ull << 63);334}335 336void event_bhrb_init(struct event *e, unsigned ifm)337{338 e->attr.config |= (1ull << 62) | ((u64)ifm << 60);339}340 341void event_leader_ebb_init(struct event *e)342{343 event_ebb_init(e);344 345 e->attr.exclusive = 1;346 e->attr.pinned = 1;347}348 349int ebb_child(union pipe read_pipe, union pipe write_pipe)350{351 struct event event;352 uint64_t val;353 354 FAIL_IF(wait_for_parent(read_pipe));355 356 event_init_named(&event, 0x1001e, "cycles");357 event_leader_ebb_init(&event);358 359 event.attr.exclude_kernel = 1;360 event.attr.exclude_hv = 1;361 event.attr.exclude_idle = 1;362 363 FAIL_IF(event_open(&event));364 365 ebb_enable_pmc_counting(1);366 setup_ebb_handler(standard_ebb_callee);367 ebb_global_enable();368 369 FAIL_IF(event_enable(&event));370 371 if (event_read(&event)) {372 /*373 * Some tests expect to fail here, so don't report an error on374 * this line, and return a distinguisable error code. Tell the375 * parent an error happened.376 */377 notify_parent_of_error(write_pipe);378 return 2;379 }380 381 mtspr(SPRN_PMC1, pmc_sample_period(sample_period));382 383 FAIL_IF(notify_parent(write_pipe));384 FAIL_IF(wait_for_parent(read_pipe));385 FAIL_IF(notify_parent(write_pipe));386 387 while (ebb_state.stats.ebb_count < 20) {388 FAIL_IF(core_busy_loop());389 390 /* To try and hit SIGILL case */391 val = mfspr(SPRN_MMCRA);392 val |= mfspr(SPRN_MMCR2);393 val |= mfspr(SPRN_MMCR0);394 }395 396 ebb_global_disable();397 ebb_freeze_pmcs();398 399 dump_ebb_state();400 401 event_close(&event);402 403 FAIL_IF(ebb_state.stats.ebb_count == 0);404 405 return 0;406}407 408static jmp_buf setjmp_env;409 410static void sigill_handler(int signal)411{412 printf("Took sigill\n");413 longjmp(setjmp_env, 1);414}415 416static struct sigaction sigill_action = {417 .sa_handler = sigill_handler,418};419 420int catch_sigill(void (*func)(void))421{422 if (sigaction(SIGILL, &sigill_action, NULL)) {423 perror("sigaction");424 return 1;425 }426 427 if (setjmp(setjmp_env) == 0) {428 func();429 return 1;430 }431 432 return 0;433}434 435void write_pmc1(void)436{437 mtspr(SPRN_PMC1, 0);438}439 440void write_pmc(int pmc, u64 value)441{442 switch (pmc) {443 case 1: mtspr(SPRN_PMC1, value); break;444 case 2: mtspr(SPRN_PMC2, value); break;445 case 3: mtspr(SPRN_PMC3, value); break;446 case 4: mtspr(SPRN_PMC4, value); break;447 case 5: mtspr(SPRN_PMC5, value); break;448 case 6: mtspr(SPRN_PMC6, value); break;449 }450}451 452u64 read_pmc(int pmc)453{454 switch (pmc) {455 case 1: return mfspr(SPRN_PMC1);456 case 2: return mfspr(SPRN_PMC2);457 case 3: return mfspr(SPRN_PMC3);458 case 4: return mfspr(SPRN_PMC4);459 case 5: return mfspr(SPRN_PMC5);460 case 6: return mfspr(SPRN_PMC6);461 }462 463 return 0;464}465 466static void term_handler(int signal)467{468 dump_summary_ebb_state();469 dump_ebb_hw_state();470 abort();471}472 473struct sigaction term_action = {474 .sa_handler = term_handler,475};476 477static void __attribute__((constructor)) ebb_init(void)478{479 clear_ebb_stats();480 481 if (sigaction(SIGTERM, &term_action, NULL))482 perror("sigaction");483 484 ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024);485}486