1590 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight4 * sink device could then pair with an appropriate per-cpu coresight source5 * device (ETE) thus generating required trace data. Trace can be enabled6 * via the perf framework.7 *8 * The AUX buffer handling is inspired from Arm SPE PMU driver.9 *10 * Copyright (C) 2020 ARM Ltd.11 *12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>13 */14#define DRVNAME "arm_trbe"15 16#define pr_fmt(fmt) DRVNAME ": " fmt17 18#include <asm/barrier.h>19#include <asm/cpufeature.h>20#include <linux/vmalloc.h>21 22#include "coresight-self-hosted-trace.h"23#include "coresight-trbe.h"24 25#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))26 27/*28 * A padding packet that will help the user space tools29 * in skipping relevant sections in the captured trace30 * data which could not be decoded. TRBE doesn't support31 * formatting the trace data, unlike the legacy CoreSight32 * sinks and thus we use ETE trace packets to pad the33 * sections of the buffer.34 */35#define ETE_IGNORE_PACKET 0x7036 37/*38 * Minimum amount of meaningful trace will contain:39 * A-Sync, Trace Info, Trace On, Address, Atom.40 * This is about 44bytes of ETE trace. To be on41 * the safer side, we assume 64bytes is the minimum42 * space required for a meaningful session, before43 * we hit a "WRAP" event.44 */45#define TRBE_TRACE_MIN_BUF_SIZE 6446 47enum trbe_fault_action {48 TRBE_FAULT_ACT_WRAP,49 TRBE_FAULT_ACT_SPURIOUS,50 TRBE_FAULT_ACT_FATAL,51};52 53struct trbe_buf {54 /*55 * Even though trbe_base represents vmap()56 * mapped allocated buffer's start address,57 * it's being as unsigned long for various58 * arithmetic and comparision operations &59 * also to be consistent with trbe_write &60 * trbe_limit sibling pointers.61 */62 unsigned long trbe_base;63 /* The base programmed into the TRBE */64 unsigned long trbe_hw_base;65 unsigned long trbe_limit;66 unsigned long trbe_write;67 int nr_pages;68 void **pages;69 bool snapshot;70 struct trbe_cpudata *cpudata;71};72 73/*74 * TRBE erratum list75 *76 * The errata are defined in arm64 generic cpu_errata framework.77 * Since the errata work arounds could be applied individually78 * to the affected CPUs inside the TRBE driver, we need to know if79 * a given CPU is affected by the erratum. Unlike the other erratum80 * work arounds, TRBE driver needs to check multiple times during81 * a trace session. Thus we need a quicker access to per-CPU82 * errata and not issue costly this_cpu_has_cap() everytime.83 * We keep a set of the affected errata in trbe_cpudata, per TRBE.84 *85 * We rely on the corresponding cpucaps to be defined for a given86 * TRBE erratum. We map the given cpucap into a TRBE internal number87 * to make the tracking of the errata lean.88 *89 * This helps in :90 * - Not duplicating the detection logic91 * - Streamlined detection of erratum across the system92 */93#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 094#define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE 195#define TRBE_NEEDS_DRAIN_AFTER_DISABLE 296#define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE 397#define TRBE_IS_BROKEN 498 99static int trbe_errata_cpucaps[] = {100 [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,101 [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,102 [TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,103 [TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,104 [TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,105 -1, /* Sentinel, must be the last entry */106};107 108/* The total number of listed errata in trbe_errata_cpucaps */109#define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1)110 111/*112 * Safe limit for the number of bytes that may be overwritten113 * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.114 */115#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256116 117/*118 * struct trbe_cpudata: TRBE instance specific data119 * @trbe_flag - TRBE dirty/access flag support120 * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.121 * @trbe_align - Software alignment used for the TRBPTR_EL1.122 * @cpu - CPU this TRBE belongs to.123 * @mode - Mode of current operation. (perf/disabled)124 * @drvdata - TRBE specific drvdata125 * @errata - Bit map for the errata on this TRBE.126 */127struct trbe_cpudata {128 bool trbe_flag;129 u64 trbe_hw_align;130 u64 trbe_align;131 int cpu;132 enum cs_mode mode;133 struct trbe_buf *buf;134 struct trbe_drvdata *drvdata;135 DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);136};137 138struct trbe_drvdata {139 struct trbe_cpudata __percpu *cpudata;140 struct perf_output_handle * __percpu *handle;141 struct hlist_node hotplug_node;142 int irq;143 cpumask_t supported_cpus;144 enum cpuhp_state trbe_online;145 struct platform_device *pdev;146};147 148static void trbe_check_errata(struct trbe_cpudata *cpudata)149{150 int i;151 152 for (i = 0; i < TRBE_ERRATA_MAX; i++) {153 int cap = trbe_errata_cpucaps[i];154 155 if (WARN_ON_ONCE(cap < 0))156 return;157 if (this_cpu_has_cap(cap))158 set_bit(i, cpudata->errata);159 }160}161 162static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)163{164 return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);165}166 167static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)168{169 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);170}171 172static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)173{174 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);175}176 177static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)178{179 /*180 * Errata affected TRBE implementation will need TSB CSYNC and181 * DSB in order to prevent subsequent writes into certain TRBE182 * system registers from being ignored and not effected.183 */184 return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);185}186 187static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)188{189 /*190 * Errata affected TRBE implementation will need an additional191 * context synchronization in order to prevent an inconsistent192 * TRBE prohibited region view on the CPU which could possibly193 * corrupt the TRBE buffer or the TRBE state.194 */195 return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);196}197 198static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)199{200 return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);201}202 203static int trbe_alloc_node(struct perf_event *event)204{205 if (event->cpu == -1)206 return NUMA_NO_NODE;207 return cpu_to_node(event->cpu);208}209 210static inline void trbe_drain_buffer(void)211{212 tsb_csync();213 dsb(nsh);214}215 216static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)217{218 /*219 * Enable the TRBE without clearing LIMITPTR which220 * might be required for fetching the buffer limits.221 */222 trblimitr |= TRBLIMITR_EL1_E;223 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);224 225 /* Synchronize the TRBE enable event */226 isb();227 228 if (trbe_needs_ctxt_sync_after_enable(cpudata))229 isb();230}231 232static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)233{234 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);235 236 /*237 * Disable the TRBE without clearing LIMITPTR which238 * might be required for fetching the buffer limits.239 */240 trblimitr &= ~TRBLIMITR_EL1_E;241 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);242 243 if (trbe_needs_drain_after_disable(cpudata))244 trbe_drain_buffer();245 isb();246}247 248static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)249{250 trbe_drain_buffer();251 set_trbe_disabled(cpudata);252}253 254static void trbe_reset_local(struct trbe_cpudata *cpudata)255{256 trbe_drain_and_disable_local(cpudata);257 write_sysreg_s(0, SYS_TRBLIMITR_EL1);258 write_sysreg_s(0, SYS_TRBPTR_EL1);259 write_sysreg_s(0, SYS_TRBBASER_EL1);260 write_sysreg_s(0, SYS_TRBSR_EL1);261}262 263static void trbe_report_wrap_event(struct perf_output_handle *handle)264{265 /*266 * Mark the buffer to indicate that there was a WRAP event by267 * setting the COLLISION flag. This indicates to the user that268 * the TRBE trace collection was stopped without stopping the269 * ETE and thus there might be some amount of trace that was270 * lost between the time the WRAP was detected and the IRQ271 * was consumed by the CPU.272 *273 * Setting the TRUNCATED flag would move the event to STOPPED274 * state unnecessarily, even when there is space left in the275 * ring buffer. Using the COLLISION flag doesn't have this side276 * effect. We only set TRUNCATED flag when there is no space277 * left in the ring buffer.278 */279 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);280}281 282static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)283{284 struct trbe_buf *buf = etm_perf_sink_config(handle);285 286 /*287 * We cannot proceed with the buffer collection and we288 * do not have any data for the current session. The289 * etm_perf driver expects to close out the aux_buffer290 * at event_stop(). So disable the TRBE here and leave291 * the update_buffer() to return a 0 size.292 */293 trbe_drain_and_disable_local(buf->cpudata);294 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);295 perf_aux_output_end(handle, 0);296 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;297}298 299/*300 * TRBE Buffer Management301 *302 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,303 * it starts writing trace data from the write pointer onward till the limit pointer.304 * When the write pointer reaches the address just before the limit pointer, it gets305 * wrapped around again to the base pointer. This is called a TRBE wrap event, which306 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver307 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ308 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and309 * LIMIT pointers.310 *311 * Wrap around with an IRQ312 * ------ < ------ < ------- < ----- < -----313 * | |314 * ------ > ------ > ------- > ----- > -----315 *316 * +---------------+-----------------------+317 * | | |318 * +---------------+-----------------------+319 * Base Pointer Write Pointer Limit Pointer320 *321 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write322 * pointer can be aligned to the implementation defined TRBE trace buffer alignment323 * as captured in trbe_cpudata->trbe_align.324 *325 *326 * head tail wakeup327 * +---------------------------------------+----- ~ ~ ------328 * |$$$$$$$|################|$$$$$$$$$$$$$$| |329 * +---------------------------------------+----- ~ ~ ------330 * Base Pointer Write Pointer Limit Pointer331 *332 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing333 * values which tracks all the driver writes and user reads from the perf auxiliary334 * buffer. Generally [head..tail] is the area where the driver can write into unless335 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and336 * configured depending on the perf_output_handle indices, so that the driver does337 * not override into areas in the perf auxiliary buffer which is being or yet to be338 * consumed from the user space. The enabled TRBE buffer area is a moving subset of339 * the allocated perf auxiliary buffer.340 */341 342static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)343{344 memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);345}346 347static void trbe_pad_buf(struct perf_output_handle *handle, int len)348{349 struct trbe_buf *buf = etm_perf_sink_config(handle);350 u64 head = PERF_IDX2OFF(handle->head, buf);351 352 __trbe_pad_buf(buf, head, len);353 if (!buf->snapshot)354 perf_aux_output_skip(handle, len);355}356 357static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)358{359 struct trbe_buf *buf = etm_perf_sink_config(handle);360 361 /*362 * The ETE trace has alignment synchronization packets allowing363 * the decoder to reset in case of an overflow or corruption.364 * So we can use the entire buffer for the snapshot mode.365 */366 return buf->nr_pages * PAGE_SIZE;367}368 369static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)370{371 u64 size = TRBE_TRACE_MIN_BUF_SIZE;372 struct trbe_buf *buf = etm_perf_sink_config(handle);373 struct trbe_cpudata *cpudata = buf->cpudata;374 375 /*376 * When the TRBE is affected by an erratum that could make it377 * write to the next "virtually addressed" page beyond the LIMIT.378 * We need to make sure there is always a PAGE after the LIMIT,379 * within the buffer. Thus we ensure there is at least an extra380 * page than normal. With this we could then adjust the LIMIT381 * pointer down by a PAGE later.382 */383 if (trbe_may_write_out_of_range(cpudata))384 size += PAGE_SIZE;385 return size;386}387 388/*389 * TRBE Limit Calculation390 *391 * The following markers are used to illustrate various TRBE buffer situations.392 *393 * $$$$ - Data area, unconsumed captured trace data, not to be overridden394 * #### - Free area, enabled, trace will be written395 * %%%% - Free area, disabled, trace will not be written396 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped397 */398static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)399{400 struct trbe_buf *buf = etm_perf_sink_config(handle);401 struct trbe_cpudata *cpudata = buf->cpudata;402 const u64 bufsize = buf->nr_pages * PAGE_SIZE;403 u64 limit = bufsize;404 u64 head, tail, wakeup;405 406 head = PERF_IDX2OFF(handle->head, buf);407 408 /*409 * head410 * ------->|411 * |412 * head TRBE align tail413 * +----|-------|---------------|-------+414 * |$$$$|=======|###############|$$$$$$$|415 * +----|-------|---------------|-------+416 * trbe_base trbe_base + nr_pages417 *418 * Perf aux buffer output head position can be misaligned depending on419 * various factors including user space reads. In case misaligned, head420 * needs to be aligned before TRBE can be configured. Pad the alignment421 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools422 * and skip this section thus advancing the head.423 */424 if (!IS_ALIGNED(head, cpudata->trbe_align)) {425 unsigned long delta = roundup(head, cpudata->trbe_align) - head;426 427 delta = min(delta, handle->size);428 trbe_pad_buf(handle, delta);429 head = PERF_IDX2OFF(handle->head, buf);430 }431 432 /*433 * head = tail (size = 0)434 * +----|-------------------------------+435 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |436 * +----|-------------------------------+437 * trbe_base trbe_base + nr_pages438 *439 * Perf aux buffer does not have any space for the driver to write into.440 */441 if (!handle->size)442 return 0;443 444 /* Compute the tail and wakeup indices now that we've aligned head */445 tail = PERF_IDX2OFF(handle->head + handle->size, buf);446 wakeup = PERF_IDX2OFF(handle->wakeup, buf);447 448 /*449 * Lets calculate the buffer area which TRBE could write into. There450 * are three possible scenarios here. Limit needs to be aligned with451 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the452 * unconsumed data.453 *454 * 1) head < tail455 *456 * head tail457 * +----|-----------------------|-------+458 * |$$$$|#######################|$$$$$$$|459 * +----|-----------------------|-------+460 * trbe_base limit trbe_base + nr_pages461 *462 * TRBE could write into [head..tail] area. Unless the tail is right at463 * the end of the buffer, neither an wrap around nor an IRQ is expected464 * while being enabled.465 *466 * 2) head == tail467 *468 * head = tail (size > 0)469 * +----|-------------------------------+470 * |%%%%|###############################|471 * +----|-------------------------------+472 * trbe_base limit = trbe_base + nr_pages473 *474 * TRBE should just write into [head..base + nr_pages] area even though475 * the entire buffer is empty. Reason being, when the trace reaches the476 * end of the buffer, it will just wrap around with an IRQ giving an477 * opportunity to reconfigure the buffer.478 *479 * 3) tail < head480 *481 * tail head482 * +----|-----------------------|-------+483 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|484 * +----|-----------------------|-------+485 * trbe_base limit = trbe_base + nr_pages486 *487 * TRBE should just write into [head..base + nr_pages] area even though488 * the [trbe_base..tail] is also empty. Reason being, when the trace489 * reaches the end of the buffer, it will just wrap around with an IRQ490 * giving an opportunity to reconfigure the buffer.491 */492 if (head < tail)493 limit = round_down(tail, PAGE_SIZE);494 495 /*496 * Wakeup may be arbitrarily far into the future. If it's not in the497 * current generation, either we'll wrap before hitting it, or it's498 * in the past and has been handled already.499 *500 * If there's a wakeup before we wrap, arrange to be woken up by the501 * page boundary following it. Keep the tail boundary if that's lower.502 *503 * head wakeup tail504 * +----|---------------|-------|-------+505 * |$$$$|###############|%%%%%%%|$$$$$$$|506 * +----|---------------|-------|-------+507 * trbe_base limit trbe_base + nr_pages508 */509 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)510 limit = min(limit, round_up(wakeup, PAGE_SIZE));511 512 /*513 * There are two situation when this can happen i.e limit is before514 * the head and hence TRBE cannot be configured.515 *516 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both517 * within the same PAGE size range.518 *519 * PAGE_SIZE520 * |----------------------|521 *522 * limit head tail523 * +------------|------|--------|-------+524 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|525 * +------------|------|--------|-------+526 * trbe_base trbe_base + nr_pages527 *528 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both529 * head and wakeup are within same PAGE size range.530 *531 * PAGE_SIZE532 * |----------------------|533 *534 * limit head wakeup tail535 * +----|------|-------|--------|-------+536 * |$$$$$$$$$$$|=======|========|$$$$$$$|537 * +----|------|-------|--------|-------+538 * trbe_base trbe_base + nr_pages539 */540 if (limit > head)541 return limit;542 543 trbe_pad_buf(handle, handle->size);544 return 0;545}546 547static unsigned long trbe_normal_offset(struct perf_output_handle *handle)548{549 struct trbe_buf *buf = etm_perf_sink_config(handle);550 u64 limit = __trbe_normal_offset(handle);551 u64 head = PERF_IDX2OFF(handle->head, buf);552 553 /*554 * If the head is too close to the limit and we don't555 * have space for a meaningful run, we rather pad it556 * and start fresh.557 *558 * We might have to do this more than once to make sure559 * we have enough required space.560 */561 while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {562 trbe_pad_buf(handle, limit - head);563 limit = __trbe_normal_offset(handle);564 head = PERF_IDX2OFF(handle->head, buf);565 }566 return limit;567}568 569static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)570{571 struct trbe_buf *buf = etm_perf_sink_config(handle);572 unsigned long offset;573 574 if (buf->snapshot)575 offset = trbe_snapshot_offset(handle);576 else577 offset = trbe_normal_offset(handle);578 return buf->trbe_base + offset;579}580 581static void clr_trbe_status(void)582{583 u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);584 585 WARN_ON(is_trbe_enabled());586 trbsr &= ~TRBSR_EL1_IRQ;587 trbsr &= ~TRBSR_EL1_TRG;588 trbsr &= ~TRBSR_EL1_WRAP;589 trbsr &= ~TRBSR_EL1_EC_MASK;590 trbsr &= ~TRBSR_EL1_BSC_MASK;591 trbsr &= ~TRBSR_EL1_S;592 write_sysreg_s(trbsr, SYS_TRBSR_EL1);593}594 595static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)596{597 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);598 unsigned long addr = buf->trbe_limit;599 600 WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT)));601 WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));602 603 trblimitr &= ~TRBLIMITR_EL1_nVM;604 trblimitr &= ~TRBLIMITR_EL1_FM_MASK;605 trblimitr &= ~TRBLIMITR_EL1_TM_MASK;606 trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;607 608 /*609 * Fill trace buffer mode is used here while configuring the610 * TRBE for trace capture. In this particular mode, the trace611 * collection is stopped and a maintenance interrupt is raised612 * when the current write pointer wraps. This pause in trace613 * collection gives the software an opportunity to capture the614 * trace data in the interrupt handler, before reconfiguring615 * the TRBE.616 */617 trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &618 TRBLIMITR_EL1_FM_MASK;619 620 /*621 * Trigger mode is not used here while configuring the TRBE for622 * the trace capture. Hence just keep this in the ignore mode.623 */624 trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &625 TRBLIMITR_EL1_TM_MASK;626 trblimitr |= (addr & PAGE_MASK);627 set_trbe_enabled(buf->cpudata, trblimitr);628}629 630static void trbe_enable_hw(struct trbe_buf *buf)631{632 WARN_ON(buf->trbe_hw_base < buf->trbe_base);633 WARN_ON(buf->trbe_write < buf->trbe_hw_base);634 WARN_ON(buf->trbe_write >= buf->trbe_limit);635 set_trbe_disabled(buf->cpudata);636 clr_trbe_status();637 set_trbe_base_pointer(buf->trbe_hw_base);638 set_trbe_write_pointer(buf->trbe_write);639 640 /*641 * Synchronize all the register updates642 * till now before enabling the TRBE.643 */644 isb();645 set_trbe_limit_pointer_enabled(buf);646}647 648static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,649 u64 trbsr)650{651 int ec = get_trbe_ec(trbsr);652 int bsc = get_trbe_bsc(trbsr);653 struct trbe_buf *buf = etm_perf_sink_config(handle);654 struct trbe_cpudata *cpudata = buf->cpudata;655 656 WARN_ON(is_trbe_running(trbsr));657 if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))658 return TRBE_FAULT_ACT_FATAL;659 660 if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))661 return TRBE_FAULT_ACT_FATAL;662 663 /*664 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,665 * it might write data after a WRAP event in the fill mode.666 * Thus the check TRBPTR == TRBBASER will not be honored.667 */668 if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&669 (trbe_may_overwrite_in_fill_mode(cpudata) ||670 get_trbe_write_pointer() == get_trbe_base_pointer()))671 return TRBE_FAULT_ACT_WRAP;672 673 return TRBE_FAULT_ACT_SPURIOUS;674}675 676static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,677 struct trbe_buf *buf, bool wrap)678{679 u64 write;680 u64 start_off, end_off;681 u64 size;682 u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;683 684 /*685 * If the TRBE has wrapped around the write pointer has686 * wrapped and should be treated as limit.687 *688 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,689 * it may write upto 64bytes beyond the "LIMIT". The driver already690 * keeps a valid page next to the LIMIT and we could potentially691 * consume the trace data that may have been collected there. But we692 * cannot be really sure it is available, and the TRBPTR may not693 * indicate the same. Also, affected cores are also affected by another694 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus695 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those696 * 64bytes. Thus we ignore the potential triggering of the erratum697 * on WRAP and limit the data to LIMIT.698 */699 if (wrap)700 write = get_trbe_limit_pointer();701 else702 write = get_trbe_write_pointer();703 704 /*705 * TRBE may use a different base address than the base706 * of the ring buffer. Thus use the beginning of the ring707 * buffer to compute the offsets.708 */709 end_off = write - buf->trbe_base;710 start_off = PERF_IDX2OFF(handle->head, buf);711 712 if (WARN_ON_ONCE(end_off < start_off))713 return 0;714 715 size = end_off - start_off;716 /*717 * If the TRBE is affected by the following erratum, we must fill718 * the space we skipped with IGNORE packets. And we are always719 * guaranteed to have at least a PAGE_SIZE space in the buffer.720 */721 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&722 !WARN_ON(size < overwrite_skip))723 __trbe_pad_buf(buf, start_off, overwrite_skip);724 725 return size;726}727 728static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,729 struct perf_event *event, void **pages,730 int nr_pages, bool snapshot)731{732 struct trbe_buf *buf;733 struct page **pglist;734 int i;735 736 /*737 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with738 * just a single page, there would not be any room left while writing739 * into a partially filled TRBE buffer after the page size alignment.740 * Hence restrict the minimum buffer size as two pages.741 */742 if (nr_pages < 2)743 return NULL;744 745 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));746 if (!buf)747 return ERR_PTR(-ENOMEM);748 749 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);750 if (!pglist) {751 kfree(buf);752 return ERR_PTR(-ENOMEM);753 }754 755 for (i = 0; i < nr_pages; i++)756 pglist[i] = virt_to_page(pages[i]);757 758 buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);759 if (!buf->trbe_base) {760 kfree(pglist);761 kfree(buf);762 return ERR_PTR(-ENOMEM);763 }764 buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;765 buf->trbe_write = buf->trbe_base;766 buf->snapshot = snapshot;767 buf->nr_pages = nr_pages;768 buf->pages = pages;769 kfree(pglist);770 return buf;771}772 773static void arm_trbe_free_buffer(void *config)774{775 struct trbe_buf *buf = config;776 777 vunmap((void *)buf->trbe_base);778 kfree(buf);779}780 781static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,782 struct perf_output_handle *handle,783 void *config)784{785 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);786 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);787 struct trbe_buf *buf = config;788 enum trbe_fault_action act;789 unsigned long size, status;790 unsigned long flags;791 bool wrap = false;792 793 WARN_ON(buf->cpudata != cpudata);794 WARN_ON(cpudata->cpu != smp_processor_id());795 WARN_ON(cpudata->drvdata != drvdata);796 if (cpudata->mode != CS_MODE_PERF)797 return 0;798 799 /*800 * We are about to disable the TRBE. And this could in turn801 * fill up the buffer triggering, an IRQ. This could be consumed802 * by the PE asynchronously, causing a race here against803 * the IRQ handler in closing out the handle. So, let us804 * make sure the IRQ can't trigger while we are collecting805 * the buffer. We also make sure that a WRAP event is handled806 * accordingly.807 */808 local_irq_save(flags);809 810 /*811 * If the TRBE was disabled due to lack of space in the AUX buffer or a812 * spurious fault, the driver leaves it disabled, truncating the buffer.813 * Since the etm_perf driver expects to close out the AUX buffer, the814 * driver skips it. Thus, just pass in 0 size here to indicate that the815 * buffer was truncated.816 */817 if (!is_trbe_enabled()) {818 size = 0;819 goto done;820 }821 /*822 * perf handle structure needs to be shared with the TRBE IRQ handler for823 * capturing trace data and restarting the handle. There is a probability824 * of an undefined reference based crash when etm event is being stopped825 * while a TRBE IRQ also getting processed. This happens due the release826 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping827 * the TRBE here will ensure that no IRQ could be generated when the perf828 * handle gets freed in etm_event_stop().829 */830 trbe_drain_and_disable_local(cpudata);831 832 /* Check if there is a pending interrupt and handle it here */833 status = read_sysreg_s(SYS_TRBSR_EL1);834 if (is_trbe_irq(status)) {835 836 /*837 * Now that we are handling the IRQ here, clear the IRQ838 * from the status, to let the irq handler know that it839 * is taken care of.840 */841 clr_trbe_irq();842 isb();843 844 act = trbe_get_fault_act(handle, status);845 /*846 * If this was not due to a WRAP event, we have some847 * errors and as such buffer is empty.848 */849 if (act != TRBE_FAULT_ACT_WRAP) {850 size = 0;851 goto done;852 }853 854 trbe_report_wrap_event(handle);855 wrap = true;856 }857 858 size = trbe_get_trace_size(handle, buf, wrap);859 860done:861 local_irq_restore(flags);862 863 if (buf->snapshot)864 handle->head += size;865 return size;866}867 868 869static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)870{871 /*872 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache873 * line size from the "TRBBASER_EL1" in the event of a "FILL".874 * Thus, we could loose some amount of the trace at the base.875 *876 * Before Fix:877 *878 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)879 * | \/ /880 * -------------------------------------------------------------881 * | Pg0 | Pg1 | | | PgN |882 * -------------------------------------------------------------883 *884 * In the normal course of action, we would set the TRBBASER to the885 * beginning of the ring-buffer (normal-BASE). But with the erratum,886 * the TRBE could overwrite the contents at the "normal-BASE", after887 * hitting the "normal-LIMIT", since it doesn't stop as expected. And888 * this is wrong. This could result in overwriting trace collected in889 * one of the previous runs, being consumed by the user. So we must890 * always make sure that the TRBBASER is within the region891 * [head, head+size]. Note that TRBBASER must be PAGE aligned,892 *893 * After moving the BASE:894 *895 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)896 * | \/ /897 * -------------------------------------------------------------898 * | | |xyzdef. |.. tuvw| |899 * -------------------------------------------------------------900 * /901 * New-BASER902 *903 * Also, we would set the TRBPTR to head (after adjusting for904 * alignment) at normal-PTR. This would mean that the last few bytes905 * of the trace (say, "xyz") might overwrite the first few bytes of906 * trace written ("abc"). More importantly they will appear in what907 * userspace sees as the beginning of the trace, which is wrong. We may908 * not always have space to move the latest trace "xyz" to the correct909 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).910 * Thus it is easier to ignore those bytes than to complicate the911 * driver to move it, assuming that the erratum was triggered and912 * doing additional checks to see if there is indeed allowed space at913 * TRBLIMITR.LIMIT.914 *915 * Thus the full workaround will move the BASE and the PTR and would916 * look like (after padding at the skipped bytes at the end of917 * session) :918 *919 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)920 * | \/ /921 * -------------------------------------------------------------922 * | | |///abc.. |.. rst| |923 * -------------------------------------------------------------924 * / |925 * New-BASER New-TRBPTR926 *927 * To summarize, with the work around:928 *929 * - We always align the offset for the next session to PAGE_SIZE930 * (This is to ensure we can program the TRBBASER to this offset931 * within the region [head...head+size]).932 *933 * - At TRBE enable:934 * - Set the TRBBASER to the page aligned offset of the current935 * proposed write offset. (which is guaranteed to be aligned936 * as above)937 * - Move the TRBPTR to skip first 256bytes (that might be938 * overwritten with the erratum). This ensures that the trace939 * generated in the session is not re-written.940 *941 * - At trace collection:942 * - Pad the 256bytes skipped above again with IGNORE packets.943 */944 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {945 if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))946 return -EINVAL;947 buf->trbe_hw_base = buf->trbe_write;948 buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;949 }950 951 /*952 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to953 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"954 * may be:955 * - The page beyond the ring buffer. This could mean, TRBE could956 * corrupt another entity (kernel / user)957 * - A portion of the "ring buffer" consumed by the userspace.958 * i.e, a page outisde [head, head + size].959 *960 * We work around this by:961 * - Making sure that we have at least an extra space of PAGE left962 * in the ring buffer [head, head + size], than we normally do963 * without the erratum. See trbe_min_trace_buf_size().964 *965 * - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside966 * the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).967 */968 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {969 s64 space = buf->trbe_limit - buf->trbe_write;970 /*971 * We must have more than a PAGE_SIZE worth space in the proposed972 * range for the TRBE.973 */974 if (WARN_ON(space <= PAGE_SIZE ||975 !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))976 return -EINVAL;977 buf->trbe_limit -= PAGE_SIZE;978 }979 980 return 0;981}982 983static int __arm_trbe_enable(struct trbe_buf *buf,984 struct perf_output_handle *handle)985{986 int ret = 0;987 988 perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);989 buf->trbe_limit = compute_trbe_buffer_limit(handle);990 buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);991 if (buf->trbe_limit == buf->trbe_base) {992 ret = -ENOSPC;993 goto err;994 }995 /* Set the base of the TRBE to the buffer base */996 buf->trbe_hw_base = buf->trbe_base;997 998 ret = trbe_apply_work_around_before_enable(buf);999 if (ret)1000 goto err;1001 1002 *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;1003 trbe_enable_hw(buf);1004 return 0;1005err:1006 trbe_stop_and_truncate_event(handle);1007 return ret;1008}1009 1010static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode,1011 void *data)1012{1013 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1014 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);1015 struct perf_output_handle *handle = data;1016 struct trbe_buf *buf = etm_perf_sink_config(handle);1017 1018 WARN_ON(cpudata->cpu != smp_processor_id());1019 WARN_ON(cpudata->drvdata != drvdata);1020 if (mode != CS_MODE_PERF)1021 return -EINVAL;1022 1023 cpudata->buf = buf;1024 cpudata->mode = mode;1025 buf->cpudata = cpudata;1026 1027 return __arm_trbe_enable(buf, handle);1028}1029 1030static int arm_trbe_disable(struct coresight_device *csdev)1031{1032 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1033 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);1034 struct trbe_buf *buf = cpudata->buf;1035 1036 WARN_ON(buf->cpudata != cpudata);1037 WARN_ON(cpudata->cpu != smp_processor_id());1038 WARN_ON(cpudata->drvdata != drvdata);1039 if (cpudata->mode != CS_MODE_PERF)1040 return -EINVAL;1041 1042 trbe_drain_and_disable_local(cpudata);1043 buf->cpudata = NULL;1044 cpudata->buf = NULL;1045 cpudata->mode = CS_MODE_DISABLED;1046 return 0;1047}1048 1049static void trbe_handle_spurious(struct perf_output_handle *handle)1050{1051 struct trbe_buf *buf = etm_perf_sink_config(handle);1052 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);1053 1054 /*1055 * If the IRQ was spurious, simply re-enable the TRBE1056 * back without modifying the buffer parameters to1057 * retain the trace collected so far.1058 */1059 set_trbe_enabled(buf->cpudata, trblimitr);1060}1061 1062static int trbe_handle_overflow(struct perf_output_handle *handle)1063{1064 struct perf_event *event = handle->event;1065 struct trbe_buf *buf = etm_perf_sink_config(handle);1066 unsigned long size;1067 struct etm_event_data *event_data;1068 1069 size = trbe_get_trace_size(handle, buf, true);1070 if (buf->snapshot)1071 handle->head += size;1072 1073 trbe_report_wrap_event(handle);1074 perf_aux_output_end(handle, size);1075 event_data = perf_aux_output_begin(handle, event);1076 if (!event_data) {1077 /*1078 * We are unable to restart the trace collection,1079 * thus leave the TRBE disabled. The etm-perf driver1080 * is able to detect this with a disconnected handle1081 * (handle->event = NULL).1082 */1083 trbe_drain_and_disable_local(buf->cpudata);1084 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;1085 return -EINVAL;1086 }1087 1088 return __arm_trbe_enable(buf, handle);1089}1090 1091static bool is_perf_trbe(struct perf_output_handle *handle)1092{1093 struct trbe_buf *buf = etm_perf_sink_config(handle);1094 struct trbe_cpudata *cpudata = buf->cpudata;1095 struct trbe_drvdata *drvdata = cpudata->drvdata;1096 int cpu = smp_processor_id();1097 1098 WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());1099 WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());1100 1101 if (cpudata->mode != CS_MODE_PERF)1102 return false;1103 1104 if (cpudata->cpu != cpu)1105 return false;1106 1107 if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))1108 return false;1109 1110 return true;1111}1112 1113static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)1114{1115 struct perf_output_handle **handle_ptr = dev;1116 struct perf_output_handle *handle = *handle_ptr;1117 struct trbe_buf *buf = etm_perf_sink_config(handle);1118 enum trbe_fault_action act;1119 u64 status;1120 bool truncated = false;1121 u64 trfcr;1122 1123 /* Reads to TRBSR_EL1 is fine when TRBE is active */1124 status = read_sysreg_s(SYS_TRBSR_EL1);1125 /*1126 * If the pending IRQ was handled by update_buffer callback1127 * we have nothing to do here.1128 */1129 if (!is_trbe_irq(status))1130 return IRQ_NONE;1131 1132 /* Prohibit the CPU from tracing before we disable the TRBE */1133 trfcr = cpu_prohibit_trace();1134 /*1135 * Ensure the trace is visible to the CPUs and1136 * any external aborts have been resolved.1137 */1138 trbe_drain_and_disable_local(buf->cpudata);1139 clr_trbe_irq();1140 isb();1141 1142 if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))1143 return IRQ_NONE;1144 1145 if (!is_perf_trbe(handle))1146 return IRQ_NONE;1147 1148 act = trbe_get_fault_act(handle, status);1149 switch (act) {1150 case TRBE_FAULT_ACT_WRAP:1151 truncated = !!trbe_handle_overflow(handle);1152 break;1153 case TRBE_FAULT_ACT_SPURIOUS:1154 trbe_handle_spurious(handle);1155 break;1156 case TRBE_FAULT_ACT_FATAL:1157 trbe_stop_and_truncate_event(handle);1158 truncated = true;1159 break;1160 }1161 1162 /*1163 * If the buffer was truncated, ensure perf callbacks1164 * have completed, which will disable the event.1165 *1166 * Otherwise, restore the trace filter controls to1167 * allow the tracing.1168 */1169 if (truncated)1170 irq_work_run();1171 else1172 write_trfcr(trfcr);1173 1174 return IRQ_HANDLED;1175}1176 1177static const struct coresight_ops_sink arm_trbe_sink_ops = {1178 .enable = arm_trbe_enable,1179 .disable = arm_trbe_disable,1180 .alloc_buffer = arm_trbe_alloc_buffer,1181 .free_buffer = arm_trbe_free_buffer,1182 .update_buffer = arm_trbe_update_buffer,1183};1184 1185static const struct coresight_ops arm_trbe_cs_ops = {1186 .sink_ops = &arm_trbe_sink_ops,1187};1188 1189static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)1190{1191 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);1192 1193 return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);1194}1195static DEVICE_ATTR_RO(align);1196 1197static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)1198{1199 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);1200 1201 return sprintf(buf, "%d\n", cpudata->trbe_flag);1202}1203static DEVICE_ATTR_RO(flag);1204 1205static struct attribute *arm_trbe_attrs[] = {1206 &dev_attr_align.attr,1207 &dev_attr_flag.attr,1208 NULL,1209};1210 1211static const struct attribute_group arm_trbe_group = {1212 .attrs = arm_trbe_attrs,1213};1214 1215static const struct attribute_group *arm_trbe_groups[] = {1216 &arm_trbe_group,1217 NULL,1218};1219 1220static void arm_trbe_enable_cpu(void *info)1221{1222 struct trbe_drvdata *drvdata = info;1223 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);1224 1225 trbe_reset_local(cpudata);1226 enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);1227}1228 1229static void arm_trbe_disable_cpu(void *info)1230{1231 struct trbe_drvdata *drvdata = info;1232 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);1233 1234 disable_percpu_irq(drvdata->irq);1235 trbe_reset_local(cpudata);1236}1237 1238 1239static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)1240{1241 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);1242 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);1243 struct coresight_desc desc = { 0 };1244 struct device *dev;1245 1246 if (WARN_ON(trbe_csdev))1247 return;1248 1249 /* If the TRBE was not probed on the CPU, we shouldn't be here */1250 if (WARN_ON(!cpudata->drvdata))1251 return;1252 1253 dev = &cpudata->drvdata->pdev->dev;1254 desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);1255 if (!desc.name)1256 goto cpu_clear;1257 /*1258 * TRBE coresight devices do not need regular connections1259 * information, as the paths get built between all percpu1260 * source and their respective percpu sink devices. Though1261 * coresight_register() expect device connections via the1262 * platform_data, which TRBE devices do not have. As they1263 * are not real ACPI devices, coresight_get_platform_data()1264 * ends up failing. Instead let's allocate a dummy zeroed1265 * coresight_platform_data structure and assign that back1266 * into the device for that purpose.1267 */1268 desc.pdata = devm_kzalloc(dev, sizeof(*desc.pdata), GFP_KERNEL);1269 if (IS_ERR(desc.pdata))1270 goto cpu_clear;1271 1272 desc.type = CORESIGHT_DEV_TYPE_SINK;1273 desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;1274 desc.ops = &arm_trbe_cs_ops;1275 desc.groups = arm_trbe_groups;1276 desc.dev = dev;1277 trbe_csdev = coresight_register(&desc);1278 if (IS_ERR(trbe_csdev))1279 goto cpu_clear;1280 1281 dev_set_drvdata(&trbe_csdev->dev, cpudata);1282 coresight_set_percpu_sink(cpu, trbe_csdev);1283 return;1284cpu_clear:1285 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);1286}1287 1288/*1289 * Must be called with preemption disabled, for trbe_check_errata().1290 */1291static void arm_trbe_probe_cpu(void *info)1292{1293 struct trbe_drvdata *drvdata = info;1294 int cpu = smp_processor_id();1295 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);1296 u64 trbidr;1297 1298 if (WARN_ON(!cpudata))1299 goto cpu_clear;1300 1301 if (!is_trbe_available()) {1302 pr_err("TRBE is not implemented on cpu %d\n", cpu);1303 goto cpu_clear;1304 }1305 1306 trbidr = read_sysreg_s(SYS_TRBIDR_EL1);1307 if (!is_trbe_programmable(trbidr)) {1308 pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);1309 goto cpu_clear;1310 }1311 1312 cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);1313 if (cpudata->trbe_hw_align > SZ_2K) {1314 pr_err("Unsupported alignment on cpu %d\n", cpu);1315 goto cpu_clear;1316 }1317 1318 /*1319 * Run the TRBE erratum checks, now that we know1320 * this instance is about to be registered.1321 */1322 trbe_check_errata(cpudata);1323 1324 if (trbe_is_broken(cpudata)) {1325 pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);1326 goto cpu_clear;1327 }1328 1329 /*1330 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,1331 * we must always program the TBRPTR_EL1, 256bytes from a page1332 * boundary, with TRBBASER_EL1 set to the page, to prevent1333 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.1334 *1335 * Thus make sure we always align our write pointer to a PAGE_SIZE,1336 * which also guarantees that we have at least a PAGE_SIZE space in1337 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip1338 * the required bytes at the base.1339 */1340 if (trbe_may_overwrite_in_fill_mode(cpudata))1341 cpudata->trbe_align = PAGE_SIZE;1342 else1343 cpudata->trbe_align = cpudata->trbe_hw_align;1344 1345 cpudata->trbe_flag = get_trbe_flag_update(trbidr);1346 cpudata->cpu = cpu;1347 cpudata->drvdata = drvdata;1348 return;1349cpu_clear:1350 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);1351}1352 1353static void arm_trbe_remove_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)1354{1355 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);1356 1357 if (trbe_csdev) {1358 coresight_unregister(trbe_csdev);1359 coresight_set_percpu_sink(cpu, NULL);1360 }1361}1362 1363static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)1364{1365 int cpu;1366 1367 drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));1368 if (!drvdata->cpudata)1369 return -ENOMEM;1370 1371 for_each_cpu(cpu, &drvdata->supported_cpus) {1372 /* If we fail to probe the CPU, let us defer it to hotplug callbacks */1373 if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))1374 continue;1375 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))1376 arm_trbe_register_coresight_cpu(drvdata, cpu);1377 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))1378 smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);1379 }1380 return 0;1381}1382 1383static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)1384{1385 int cpu;1386 1387 for_each_cpu(cpu, &drvdata->supported_cpus) {1388 smp_call_function_single(cpu, arm_trbe_disable_cpu, drvdata, 1);1389 arm_trbe_remove_coresight_cpu(drvdata, cpu);1390 }1391 free_percpu(drvdata->cpudata);1392 return 0;1393}1394 1395static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)1396{1397 preempt_disable();1398 arm_trbe_probe_cpu(drvdata);1399 preempt_enable();1400}1401 1402static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)1403{1404 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);1405 1406 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {1407 1408 /*1409 * If this CPU was not probed for TRBE,1410 * initialize it now.1411 */1412 if (!coresight_get_percpu_sink(cpu)) {1413 arm_trbe_probe_hotplugged_cpu(drvdata);1414 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))1415 arm_trbe_register_coresight_cpu(drvdata, cpu);1416 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))1417 arm_trbe_enable_cpu(drvdata);1418 } else {1419 arm_trbe_enable_cpu(drvdata);1420 }1421 }1422 return 0;1423}1424 1425static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)1426{1427 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);1428 1429 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))1430 arm_trbe_disable_cpu(drvdata);1431 return 0;1432}1433 1434static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)1435{1436 enum cpuhp_state trbe_online;1437 int ret;1438 1439 trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,1440 arm_trbe_cpu_startup, arm_trbe_cpu_teardown);1441 if (trbe_online < 0)1442 return trbe_online;1443 1444 ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);1445 if (ret) {1446 cpuhp_remove_multi_state(trbe_online);1447 return ret;1448 }1449 drvdata->trbe_online = trbe_online;1450 return 0;1451}1452 1453static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)1454{1455 cpuhp_state_remove_instance(drvdata->trbe_online, &drvdata->hotplug_node);1456 cpuhp_remove_multi_state(drvdata->trbe_online);1457}1458 1459static int arm_trbe_probe_irq(struct platform_device *pdev,1460 struct trbe_drvdata *drvdata)1461{1462 int ret;1463 1464 drvdata->irq = platform_get_irq(pdev, 0);1465 if (drvdata->irq < 0) {1466 pr_err("IRQ not found for the platform device\n");1467 return drvdata->irq;1468 }1469 1470 if (!irq_is_percpu(drvdata->irq)) {1471 pr_err("IRQ is not a PPI\n");1472 return -EINVAL;1473 }1474 1475 if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))1476 return -EINVAL;1477 1478 drvdata->handle = alloc_percpu(struct perf_output_handle *);1479 if (!drvdata->handle)1480 return -ENOMEM;1481 1482 ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);1483 if (ret) {1484 free_percpu(drvdata->handle);1485 return ret;1486 }1487 return 0;1488}1489 1490static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)1491{1492 free_percpu_irq(drvdata->irq, drvdata->handle);1493 free_percpu(drvdata->handle);1494}1495 1496static int arm_trbe_device_probe(struct platform_device *pdev)1497{1498 struct trbe_drvdata *drvdata;1499 struct device *dev = &pdev->dev;1500 int ret;1501 1502 /* Trace capture is not possible with kernel page table isolation */1503 if (arm64_kernel_unmapped_at_el0()) {1504 pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");1505 return -EOPNOTSUPP;1506 }1507 1508 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);1509 if (!drvdata)1510 return -ENOMEM;1511 1512 dev_set_drvdata(dev, drvdata);1513 drvdata->pdev = pdev;1514 ret = arm_trbe_probe_irq(pdev, drvdata);1515 if (ret)1516 return ret;1517 1518 ret = arm_trbe_probe_coresight(drvdata);1519 if (ret)1520 goto probe_failed;1521 1522 ret = arm_trbe_probe_cpuhp(drvdata);1523 if (ret)1524 goto cpuhp_failed;1525 1526 return 0;1527cpuhp_failed:1528 arm_trbe_remove_coresight(drvdata);1529probe_failed:1530 arm_trbe_remove_irq(drvdata);1531 return ret;1532}1533 1534static void arm_trbe_device_remove(struct platform_device *pdev)1535{1536 struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);1537 1538 arm_trbe_remove_cpuhp(drvdata);1539 arm_trbe_remove_coresight(drvdata);1540 arm_trbe_remove_irq(drvdata);1541}1542 1543static const struct of_device_id arm_trbe_of_match[] = {1544 { .compatible = "arm,trace-buffer-extension"},1545 {},1546};1547MODULE_DEVICE_TABLE(of, arm_trbe_of_match);1548 1549#ifdef CONFIG_ACPI1550static const struct platform_device_id arm_trbe_acpi_match[] = {1551 { ARMV8_TRBE_PDEV_NAME, 0 },1552 { }1553};1554MODULE_DEVICE_TABLE(platform, arm_trbe_acpi_match);1555#endif1556 1557static struct platform_driver arm_trbe_driver = {1558 .id_table = ACPI_PTR(arm_trbe_acpi_match),1559 .driver = {1560 .name = DRVNAME,1561 .of_match_table = of_match_ptr(arm_trbe_of_match),1562 .suppress_bind_attrs = true,1563 },1564 .probe = arm_trbe_device_probe,1565 .remove_new = arm_trbe_device_remove,1566};1567 1568static int __init arm_trbe_init(void)1569{1570 int ret;1571 1572 ret = platform_driver_register(&arm_trbe_driver);1573 if (!ret)1574 return 0;1575 1576 pr_err("Error registering %s platform driver\n", DRVNAME);1577 return ret;1578}1579 1580static void __exit arm_trbe_exit(void)1581{1582 platform_driver_unregister(&arm_trbe_driver);1583}1584module_init(arm_trbe_init);1585module_exit(arm_trbe_exit);1586 1587MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");1588MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");1589MODULE_LICENSE("GPL v2");1590