2454 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (c) 2014, The Linux Foundation. All rights reserved.4 */5 6#include <linux/acpi.h>7#include <linux/bitops.h>8#include <linux/kernel.h>9#include <linux/moduleparam.h>10#include <linux/init.h>11#include <linux/types.h>12#include <linux/device.h>13#include <linux/io.h>14#include <linux/err.h>15#include <linux/fs.h>16#include <linux/slab.h>17#include <linux/delay.h>18#include <linux/smp.h>19#include <linux/sysfs.h>20#include <linux/stat.h>21#include <linux/clk.h>22#include <linux/cpu.h>23#include <linux/cpu_pm.h>24#include <linux/coresight.h>25#include <linux/coresight-pmu.h>26#include <linux/pm_wakeup.h>27#include <linux/amba/bus.h>28#include <linux/seq_file.h>29#include <linux/uaccess.h>30#include <linux/perf_event.h>31#include <linux/platform_device.h>32#include <linux/pm_runtime.h>33#include <linux/property.h>34#include <linux/clk/clk-conf.h>35 36#include <asm/barrier.h>37#include <asm/sections.h>38#include <asm/sysreg.h>39#include <asm/local.h>40#include <asm/virt.h>41 42#include "coresight-etm4x.h"43#include "coresight-etm-perf.h"44#include "coresight-etm4x-cfg.h"45#include "coresight-self-hosted-trace.h"46#include "coresight-syscfg.h"47#include "coresight-trace-id.h"48 49static int boot_enable;50module_param(boot_enable, int, 0444);51MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");52 53#define PARAM_PM_SAVE_FIRMWARE 0 /* save self-hosted state as per firmware */54#define PARAM_PM_SAVE_NEVER 1 /* never save any state */55#define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */56 57static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;58module_param(pm_save_enable, int, 0444);59MODULE_PARM_DESC(pm_save_enable,60 "Save/restore state on power down: 1 = never, 2 = self-hosted");61 62static struct etmv4_drvdata *etmdrvdata[NR_CPUS];63static void etm4_set_default_config(struct etmv4_config *config);64static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,65 struct perf_event *event);66static u64 etm4_get_access_type(struct etmv4_config *config);67 68static enum cpuhp_state hp_online;69 70struct etm4_init_arg {71 struct device *dev;72 struct csdev_access *csa;73};74 75static DEFINE_PER_CPU(struct etm4_init_arg *, delayed_probe);76static int etm4_probe_cpu(unsigned int cpu);77 78/*79 * Check if TRCSSPCICRn(i) is implemented for a given instance.80 *81 * TRCSSPCICRn is implemented only if :82 * TRCSSPCICR<n> is present only if all of the following are true:83 * TRCIDR4.NUMSSCC > n.84 * TRCIDR4.NUMPC > 0b0000 .85 * TRCSSCSR<n>.PC == 0b186 */87static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n)88{89 return (n < drvdata->nr_ss_cmp) &&90 drvdata->nr_pe &&91 (drvdata->config.ss_status[n] & TRCSSCSRn_PC);92}93 94u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit)95{96 u64 res = 0;97 98 switch (offset) {99 ETM4x_READ_SYSREG_CASES(res)100 default :101 pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n",102 offset);103 }104 105 if (!_relaxed)106 __io_ar(res); /* Imitate the !relaxed I/O helpers */107 108 return res;109}110 111void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit)112{113 if (!_relaxed)114 __io_bw(); /* Imitate the !relaxed I/O helpers */115 if (!_64bit)116 val &= GENMASK(31, 0);117 118 switch (offset) {119 ETM4x_WRITE_SYSREG_CASES(val)120 default :121 pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n",122 offset);123 }124}125 126static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit)127{128 u64 res = 0;129 130 switch (offset) {131 ETE_READ_CASES(res)132 default :133 pr_warn_ratelimited("ete: trying to read unsupported register @%x\n",134 offset);135 }136 137 if (!_relaxed)138 __io_ar(res); /* Imitate the !relaxed I/O helpers */139 140 return res;141}142 143static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit)144{145 if (!_relaxed)146 __io_bw(); /* Imitate the !relaxed I/O helpers */147 if (!_64bit)148 val &= GENMASK(31, 0);149 150 switch (offset) {151 ETE_WRITE_CASES(val)152 default :153 pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n",154 offset);155 }156}157 158static void etm_detect_os_lock(struct etmv4_drvdata *drvdata,159 struct csdev_access *csa)160{161 u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR);162 163 drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr);164}165 166static void etm_write_os_lock(struct etmv4_drvdata *drvdata,167 struct csdev_access *csa, u32 val)168{169 val = !!val;170 171 switch (drvdata->os_lock_model) {172 case ETM_OSLOCK_PRESENT:173 etm4x_relaxed_write32(csa, val, TRCOSLAR);174 break;175 case ETM_OSLOCK_PE:176 write_sysreg_s(val, SYS_OSLAR_EL1);177 break;178 default:179 pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n",180 smp_processor_id(), drvdata->os_lock_model);181 fallthrough;182 case ETM_OSLOCK_NI:183 return;184 }185 isb();186}187 188static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata,189 struct csdev_access *csa)190{191 WARN_ON(drvdata->cpu != smp_processor_id());192 193 /* Writing 0 to OS Lock unlocks the trace unit registers */194 etm_write_os_lock(drvdata, csa, 0x0);195 drvdata->os_unlock = true;196}197 198static void etm4_os_unlock(struct etmv4_drvdata *drvdata)199{200 if (!WARN_ON(!drvdata->csdev))201 etm4_os_unlock_csa(drvdata, &drvdata->csdev->access);202}203 204static void etm4_os_lock(struct etmv4_drvdata *drvdata)205{206 if (WARN_ON(!drvdata->csdev))207 return;208 /* Writing 0x1 to OS Lock locks the trace registers */209 etm_write_os_lock(drvdata, &drvdata->csdev->access, 0x1);210 drvdata->os_unlock = false;211}212 213static void etm4_cs_lock(struct etmv4_drvdata *drvdata,214 struct csdev_access *csa)215{216 /* Software Lock is only accessible via memory mapped interface */217 if (csa->io_mem)218 CS_LOCK(csa->base);219}220 221static void etm4_cs_unlock(struct etmv4_drvdata *drvdata,222 struct csdev_access *csa)223{224 if (csa->io_mem)225 CS_UNLOCK(csa->base);226}227 228static int etm4_cpu_id(struct coresight_device *csdev)229{230 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);231 232 return drvdata->cpu;233}234 235int etm4_read_alloc_trace_id(struct etmv4_drvdata *drvdata)236{237 int trace_id;238 239 /*240 * This will allocate a trace ID to the cpu,241 * or return the one currently allocated.242 * The trace id function has its own lock243 */244 trace_id = coresight_trace_id_get_cpu_id(drvdata->cpu);245 if (IS_VALID_CS_TRACE_ID(trace_id))246 drvdata->trcid = (u8)trace_id;247 else248 dev_err(&drvdata->csdev->dev,249 "Failed to allocate trace ID for %s on CPU%d\n",250 dev_name(&drvdata->csdev->dev), drvdata->cpu);251 return trace_id;252}253 254void etm4_release_trace_id(struct etmv4_drvdata *drvdata)255{256 coresight_trace_id_put_cpu_id(drvdata->cpu);257}258 259struct etm4_enable_arg {260 struct etmv4_drvdata *drvdata;261 int rc;262};263 264/*265 * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs.266 * When the CPU supports FEAT_TRF, we could move the ETM to a trace267 * prohibited state by filtering the Exception levels via TRFCR_EL1.268 */269static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata)270{271 /* If the CPU doesn't support FEAT_TRF, nothing to do */272 if (!drvdata->trfcr)273 return;274 cpu_prohibit_trace();275}276 277/*278 * etm4x_allow_trace - Allow CPU tracing in the respective ELs,279 * as configured by the drvdata->config.mode for the current280 * session. Even though we have TRCVICTLR bits to filter the281 * trace in the ELs, it doesn't prevent the ETM from generating282 * a packet (e.g, TraceInfo) that might contain the addresses from283 * the excluded levels. Thus we use the additional controls provided284 * via the Trace Filtering controls (FEAT_TRF) to make sure no trace285 * is generated for the excluded ELs.286 */287static void etm4x_allow_trace(struct etmv4_drvdata *drvdata)288{289 u64 trfcr = drvdata->trfcr;290 291 /* If the CPU doesn't support FEAT_TRF, nothing to do */292 if (!trfcr)293 return;294 295 if (drvdata->config.mode & ETM_MODE_EXCL_KERN)296 trfcr &= ~TRFCR_ELx_ExTRE;297 if (drvdata->config.mode & ETM_MODE_EXCL_USER)298 trfcr &= ~TRFCR_ELx_E0TRE;299 300 write_trfcr(trfcr);301}302 303#ifdef CONFIG_ETM4X_IMPDEF_FEATURE304 305#define HISI_HIP08_AMBA_ID 0x000b6d01306#define ETM4_AMBA_MASK 0xfffff307#define HISI_HIP08_CORE_COMMIT_MASK 0x3000308#define HISI_HIP08_CORE_COMMIT_SHIFT 12309#define HISI_HIP08_CORE_COMMIT_FULL 0b00310#define HISI_HIP08_CORE_COMMIT_LVL_1 0b01311#define HISI_HIP08_CORE_COMMIT_REG sys_reg(3, 1, 15, 2, 5)312 313struct etm4_arch_features {314 void (*arch_callback)(bool enable);315};316 317static bool etm4_hisi_match_pid(unsigned int id)318{319 return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID;320}321 322static void etm4_hisi_config_core_commit(bool enable)323{324 u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 :325 HISI_HIP08_CORE_COMMIT_FULL;326 u64 val;327 328 /*329 * bit 12 and 13 of HISI_HIP08_CORE_COMMIT_REG are used together330 * to set core-commit, 2'b00 means cpu is at full speed, 2'b01,331 * 2'b10, 2'b11 mean reduce pipeline speed, and 2'b01 means level-1332 * speed(minimun value). So bit 12 and 13 should be cleared together.333 */334 val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG);335 val &= ~HISI_HIP08_CORE_COMMIT_MASK;336 val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT;337 write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG);338}339 340static struct etm4_arch_features etm4_features[] = {341 [ETM4_IMPDEF_HISI_CORE_COMMIT] = {342 .arch_callback = etm4_hisi_config_core_commit,343 },344 {},345};346 347static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)348{349 struct etm4_arch_features *ftr;350 int bit;351 352 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {353 ftr = &etm4_features[bit];354 355 if (ftr->arch_callback)356 ftr->arch_callback(true);357 }358}359 360static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)361{362 struct etm4_arch_features *ftr;363 int bit;364 365 for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {366 ftr = &etm4_features[bit];367 368 if (ftr->arch_callback)369 ftr->arch_callback(false);370 }371}372 373static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,374 struct csdev_access *csa)375{376 /*377 * TRCPIDR* registers are not required for ETMs with system378 * instructions. They must be identified by the MIDR+REVIDRs.379 * Skip the TRCPID checks for now.380 */381 if (!csa->io_mem)382 return;383 384 if (etm4_hisi_match_pid(coresight_get_pid(csa)))385 set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features);386}387#else388static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)389{390}391 392static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)393{394}395 396static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,397 struct csdev_access *csa)398{399}400#endif /* CONFIG_ETM4X_IMPDEF_FEATURE */401 402static int etm4_enable_hw(struct etmv4_drvdata *drvdata)403{404 int i, rc;405 struct etmv4_config *config = &drvdata->config;406 struct coresight_device *csdev = drvdata->csdev;407 struct device *etm_dev = &csdev->dev;408 struct csdev_access *csa = &csdev->access;409 410 411 etm4_cs_unlock(drvdata, csa);412 etm4_enable_arch_specific(drvdata);413 414 etm4_os_unlock(drvdata);415 416 rc = coresight_claim_device_unlocked(csdev);417 if (rc)418 goto done;419 420 /* Disable the trace unit before programming trace registers */421 etm4x_relaxed_write32(csa, 0, TRCPRGCTLR);422 423 /*424 * If we use system instructions, we need to synchronize the425 * write to the TRCPRGCTLR, before accessing the TRCSTATR.426 * See ARM IHI0064F, section427 * "4.3.7 Synchronization of register updates"428 */429 if (!csa->io_mem)430 isb();431 432 /* wait for TRCSTATR.IDLE to go up */433 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1))434 dev_err(etm_dev,435 "timeout while waiting for Idle Trace Status\n");436 if (drvdata->nr_pe)437 etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR);438 etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR);439 /* nothing specific implemented */440 etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR);441 etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R);442 etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R);443 if (drvdata->stallctl)444 etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR);445 etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR);446 etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR);447 etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR);448 etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR);449 etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR);450 etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR);451 etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR);452 etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR);453 if (drvdata->nr_pe_cmp)454 etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR);455 for (i = 0; i < drvdata->nrseqstate - 1; i++)456 etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i));457 if (drvdata->nrseqstate) {458 etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR);459 etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR);460 }461 etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR);462 for (i = 0; i < drvdata->nr_cntr; i++) {463 etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i));464 etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i));465 etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i));466 }467 468 /*469 * Resource selector pair 0 is always implemented and reserved. As470 * such start at 2.471 */472 for (i = 2; i < drvdata->nr_resource * 2; i++)473 etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i));474 475 for (i = 0; i < drvdata->nr_ss_cmp; i++) {476 /* always clear status bit on restart if using single-shot */477 if (config->ss_ctrl[i] || config->ss_pe_cmp[i])478 config->ss_status[i] &= ~TRCSSCSRn_STATUS;479 etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i));480 etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i));481 if (etm4x_sspcicrn_present(drvdata, i))482 etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i));483 }484 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {485 etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i));486 etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i));487 }488 for (i = 0; i < drvdata->numcidc; i++)489 etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i));490 etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0);491 if (drvdata->numcidc > 4)492 etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1);493 494 for (i = 0; i < drvdata->numvmidc; i++)495 etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i));496 etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0);497 if (drvdata->numvmidc > 4)498 etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1);499 500 if (!drvdata->skip_power_up) {501 u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR);502 503 /*504 * Request to keep the trace unit powered and also505 * emulation of powerdown506 */507 etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR);508 }509 510 /*511 * ETE mandates that the TRCRSR is written to before512 * enabling it.513 */514 if (etm4x_is_ete(drvdata))515 etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR);516 517 etm4x_allow_trace(drvdata);518 /* Enable the trace unit */519 etm4x_relaxed_write32(csa, 1, TRCPRGCTLR);520 521 /* Synchronize the register updates for sysreg access */522 if (!csa->io_mem)523 isb();524 525 /* wait for TRCSTATR.IDLE to go back down to '0' */526 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0))527 dev_err(etm_dev,528 "timeout while waiting for Idle Trace Status\n");529 530 /*531 * As recommended by section 4.3.7 ("Synchronization when using the532 * memory-mapped interface") of ARM IHI 0064D533 */534 dsb(sy);535 isb();536 537done:538 etm4_cs_lock(drvdata, csa);539 540 dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n",541 drvdata->cpu, rc);542 return rc;543}544 545static void etm4_enable_hw_smp_call(void *info)546{547 struct etm4_enable_arg *arg = info;548 549 if (WARN_ON(!arg))550 return;551 arg->rc = etm4_enable_hw(arg->drvdata);552}553 554/*555 * The goal of function etm4_config_timestamp_event() is to configure a556 * counter that will tell the tracer to emit a timestamp packet when it557 * reaches zero. This is done in order to get a more fine grained idea558 * of when instructions are executed so that they can be correlated559 * with execution on other CPUs.560 *561 * To do this the counter itself is configured to self reload and562 * TRCRSCTLR1 (always true) used to get the counter to decrement. From563 * there a resource selector is configured with the counter and the564 * timestamp control register to use the resource selector to trigger the565 * event that will insert a timestamp packet in the stream.566 */567static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata)568{569 int ctridx, ret = -EINVAL;570 int counter, rselector;571 u32 val = 0;572 struct etmv4_config *config = &drvdata->config;573 574 /* No point in trying if we don't have at least one counter */575 if (!drvdata->nr_cntr)576 goto out;577 578 /* Find a counter that hasn't been initialised */579 for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++)580 if (config->cntr_val[ctridx] == 0)581 break;582 583 /* All the counters have been configured already, bail out */584 if (ctridx == drvdata->nr_cntr) {585 pr_debug("%s: no available counter found\n", __func__);586 ret = -ENOSPC;587 goto out;588 }589 590 /*591 * Searching for an available resource selector to use, starting at592 * '2' since every implementation has at least 2 resource selector.593 * ETMIDR4 gives the number of resource selector _pairs_,594 * hence multiply by 2.595 */596 for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++)597 if (!config->res_ctrl[rselector])598 break;599 600 if (rselector == drvdata->nr_resource * 2) {601 pr_debug("%s: no available resource selector found\n",602 __func__);603 ret = -ENOSPC;604 goto out;605 }606 607 /* Remember what counter we used */608 counter = 1 << ctridx;609 610 /*611 * Initialise original and reload counter value to the smallest612 * possible value in order to get as much precision as we can.613 */614 config->cntr_val[ctridx] = 1;615 config->cntrldvr[ctridx] = 1;616 617 /* Set the trace counter control register */618 val = 0x1 << 16 | /* Bit 16, reload counter automatically */619 0x0 << 7 | /* Select single resource selector */620 0x1; /* Resource selector 1, i.e always true */621 622 config->cntr_ctrl[ctridx] = val;623 624 val = 0x2 << 16 | /* Group 0b0010 - Counter and sequencers */625 counter << 0; /* Counter to use */626 627 config->res_ctrl[rselector] = val;628 629 val = 0x0 << 7 | /* Select single resource selector */630 rselector; /* Resource selector */631 632 config->ts_ctrl = val;633 634 ret = 0;635out:636 return ret;637}638 639static int etm4_parse_event_config(struct coresight_device *csdev,640 struct perf_event *event)641{642 int ret = 0;643 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);644 struct etmv4_config *config = &drvdata->config;645 struct perf_event_attr *attr = &event->attr;646 unsigned long cfg_hash;647 int preset, cc_threshold;648 649 /* Clear configuration from previous run */650 memset(config, 0, sizeof(struct etmv4_config));651 652 if (attr->exclude_kernel)653 config->mode = ETM_MODE_EXCL_KERN;654 655 if (attr->exclude_user)656 config->mode = ETM_MODE_EXCL_USER;657 658 /* Always start from the default config */659 etm4_set_default_config(config);660 661 /* Configure filters specified on the perf cmd line, if any. */662 ret = etm4_set_event_filters(drvdata, event);663 if (ret)664 goto out;665 666 /* Go from generic option to ETMv4 specifics */667 if (attr->config & BIT(ETM_OPT_CYCACC)) {668 config->cfg |= TRCCONFIGR_CCI;669 /* TRM: Must program this for cycacc to work */670 cc_threshold = attr->config3 & ETM_CYC_THRESHOLD_MASK;671 if (!cc_threshold)672 cc_threshold = ETM_CYC_THRESHOLD_DEFAULT;673 if (cc_threshold < drvdata->ccitmin)674 cc_threshold = drvdata->ccitmin;675 config->ccctlr = cc_threshold;676 }677 if (attr->config & BIT(ETM_OPT_TS)) {678 /*679 * Configure timestamps to be emitted at regular intervals in680 * order to correlate instructions executed on different CPUs681 * (CPU-wide trace scenarios).682 */683 ret = etm4_config_timestamp_event(drvdata);684 685 /*686 * No need to go further if timestamp intervals can't687 * be configured.688 */689 if (ret)690 goto out;691 692 /* bit[11], Global timestamp tracing bit */693 config->cfg |= TRCCONFIGR_TS;694 }695 696 /* Only trace contextID when runs in root PID namespace */697 if ((attr->config & BIT(ETM_OPT_CTXTID)) &&698 task_is_in_init_pid_ns(current))699 /* bit[6], Context ID tracing bit */700 config->cfg |= TRCCONFIGR_CID;701 702 /*703 * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID704 * for recording CONTEXTIDR_EL2. Do not enable VMID tracing if the705 * kernel is not running in EL2.706 */707 if (attr->config & BIT(ETM_OPT_CTXTID2)) {708 if (!is_kernel_in_hyp_mode()) {709 ret = -EINVAL;710 goto out;711 }712 /* Only trace virtual contextID when runs in root PID namespace */713 if (task_is_in_init_pid_ns(current))714 config->cfg |= TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT;715 }716 717 /* return stack - enable if selected and supported */718 if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)719 /* bit[12], Return stack enable bit */720 config->cfg |= TRCCONFIGR_RS;721 722 /*723 * Set any selected configuration and preset.724 *725 * This extracts the values of PMU_FORMAT_ATTR(configid) and PMU_FORMAT_ATTR(preset)726 * in the perf attributes defined in coresight-etm-perf.c.727 * configid uses bits 63:32 of attr->config2, preset uses bits 3:0 of attr->config.728 * A zero configid means no configuration active, preset = 0 means no preset selected.729 */730 if (attr->config2 & GENMASK_ULL(63, 32)) {731 cfg_hash = (u32)(attr->config2 >> 32);732 preset = attr->config & 0xF;733 ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset);734 }735 736 /* branch broadcast - enable if selected and supported */737 if (attr->config & BIT(ETM_OPT_BRANCH_BROADCAST)) {738 if (!drvdata->trcbb) {739 /*740 * Missing BB support could cause silent decode errors741 * so fail to open if it's not supported.742 */743 ret = -EINVAL;744 goto out;745 } else {746 config->cfg |= BIT(ETM4_CFG_BIT_BB);747 }748 }749 750out:751 return ret;752}753 754static int etm4_enable_perf(struct coresight_device *csdev,755 struct perf_event *event,756 struct coresight_trace_id_map *id_map)757{758 int ret = 0, trace_id;759 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);760 761 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {762 ret = -EINVAL;763 goto out;764 }765 766 /* Configure the tracer based on the session's specifics */767 ret = etm4_parse_event_config(csdev, event);768 if (ret)769 goto out;770 771 /*772 * perf allocates cpu ids as part of _setup_aux() - device needs to use773 * the allocated ID. This reads the current version without allocation.774 *775 * This does not use the trace id lock to prevent lock_dep issues776 * with perf locks - we know the ID cannot change until perf shuts down777 * the session778 */779 trace_id = coresight_trace_id_read_cpu_id_map(drvdata->cpu, id_map);780 if (!IS_VALID_CS_TRACE_ID(trace_id)) {781 dev_err(&drvdata->csdev->dev, "Failed to set trace ID for %s on CPU%d\n",782 dev_name(&drvdata->csdev->dev), drvdata->cpu);783 ret = -EINVAL;784 goto out;785 }786 drvdata->trcid = (u8)trace_id;787 788 /* And enable it */789 ret = etm4_enable_hw(drvdata);790 791out:792 return ret;793}794 795static int etm4_enable_sysfs(struct coresight_device *csdev)796{797 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);798 struct etm4_enable_arg arg = { };799 unsigned long cfg_hash;800 int ret, preset;801 802 /* enable any config activated by configfs */803 cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset);804 if (cfg_hash) {805 ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset);806 if (ret)807 return ret;808 }809 810 spin_lock(&drvdata->spinlock);811 812 /* sysfs needs to read and allocate a trace ID */813 ret = etm4_read_alloc_trace_id(drvdata);814 if (ret < 0)815 goto unlock_sysfs_enable;816 817 /*818 * Executing etm4_enable_hw on the cpu whose ETM is being enabled819 * ensures that register writes occur when cpu is powered.820 */821 arg.drvdata = drvdata;822 ret = smp_call_function_single(drvdata->cpu,823 etm4_enable_hw_smp_call, &arg, 1);824 if (!ret)825 ret = arg.rc;826 if (!ret)827 drvdata->sticky_enable = true;828 829 if (ret)830 etm4_release_trace_id(drvdata);831 832unlock_sysfs_enable:833 spin_unlock(&drvdata->spinlock);834 835 if (!ret)836 dev_dbg(&csdev->dev, "ETM tracing enabled\n");837 return ret;838}839 840static int etm4_enable(struct coresight_device *csdev, struct perf_event *event,841 enum cs_mode mode, struct coresight_trace_id_map *id_map)842{843 int ret;844 845 if (!coresight_take_mode(csdev, mode)) {846 /* Someone is already using the tracer */847 return -EBUSY;848 }849 850 switch (mode) {851 case CS_MODE_SYSFS:852 ret = etm4_enable_sysfs(csdev);853 break;854 case CS_MODE_PERF:855 ret = etm4_enable_perf(csdev, event, id_map);856 break;857 default:858 ret = -EINVAL;859 }860 861 /* The tracer didn't start */862 if (ret)863 coresight_set_mode(csdev, CS_MODE_DISABLED);864 865 return ret;866}867 868static void etm4_disable_hw(void *info)869{870 u32 control;871 struct etmv4_drvdata *drvdata = info;872 struct etmv4_config *config = &drvdata->config;873 struct coresight_device *csdev = drvdata->csdev;874 struct device *etm_dev = &csdev->dev;875 struct csdev_access *csa = &csdev->access;876 int i;877 878 etm4_cs_unlock(drvdata, csa);879 etm4_disable_arch_specific(drvdata);880 881 if (!drvdata->skip_power_up) {882 /* power can be removed from the trace unit now */883 control = etm4x_relaxed_read32(csa, TRCPDCR);884 control &= ~TRCPDCR_PU;885 etm4x_relaxed_write32(csa, control, TRCPDCR);886 }887 888 control = etm4x_relaxed_read32(csa, TRCPRGCTLR);889 890 /* EN, bit[0] Trace unit enable bit */891 control &= ~0x1;892 893 /*894 * If the CPU supports v8.4 Trace filter Control,895 * set the ETM to trace prohibited region.896 */897 etm4x_prohibit_trace(drvdata);898 /*899 * Make sure everything completes before disabling, as recommended900 * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register,901 * SSTATUS") of ARM IHI 0064D902 */903 dsb(sy);904 isb();905 /* Trace synchronization barrier, is a nop if not supported */906 tsb_csync();907 etm4x_relaxed_write32(csa, control, TRCPRGCTLR);908 909 /* wait for TRCSTATR.PMSTABLE to go to '1' */910 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1))911 dev_err(etm_dev,912 "timeout while waiting for PM stable Trace Status\n");913 /* read the status of the single shot comparators */914 for (i = 0; i < drvdata->nr_ss_cmp; i++) {915 config->ss_status[i] =916 etm4x_relaxed_read32(csa, TRCSSCSRn(i));917 }918 919 /* read back the current counter values */920 for (i = 0; i < drvdata->nr_cntr; i++) {921 config->cntr_val[i] =922 etm4x_relaxed_read32(csa, TRCCNTVRn(i));923 }924 925 coresight_disclaim_device_unlocked(csdev);926 etm4_cs_lock(drvdata, csa);927 928 dev_dbg(&drvdata->csdev->dev,929 "cpu: %d disable smp call done\n", drvdata->cpu);930}931 932static int etm4_disable_perf(struct coresight_device *csdev,933 struct perf_event *event)934{935 u32 control;936 struct etm_filters *filters = event->hw.addr_filters;937 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);938 struct perf_event_attr *attr = &event->attr;939 940 if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))941 return -EINVAL;942 943 etm4_disable_hw(drvdata);944 /*945 * The config_id occupies bits 63:32 of the config2 perf event attr946 * field. If this is non-zero then we will have enabled a config.947 */948 if (attr->config2 & GENMASK_ULL(63, 32))949 cscfg_csdev_disable_active_config(csdev);950 951 /*952 * Check if the start/stop logic was active when the unit was stopped.953 * That way we can re-enable the start/stop logic when the process is954 * scheduled again. Configuration of the start/stop logic happens in955 * function etm4_set_event_filters().956 */957 control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR);958 /* TRCVICTLR::SSSTATUS, bit[9] */959 filters->ssstatus = (control & BIT(9));960 961 /*962 * perf will release trace ids when _free_aux() is963 * called at the end of the session.964 */965 966 return 0;967}968 969static void etm4_disable_sysfs(struct coresight_device *csdev)970{971 struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);972 973 /*974 * Taking hotplug lock here protects from clocks getting disabled975 * with tracing being left on (crash scenario) if user disable occurs976 * after cpu online mask indicates the cpu is offline but before the977 * DYING hotplug callback is serviced by the ETM driver.978 */979 cpus_read_lock();980 spin_lock(&drvdata->spinlock);981 982 /*983 * Executing etm4_disable_hw on the cpu whose ETM is being disabled984 * ensures that register writes occur when cpu is powered.985 */986 smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);987 988 spin_unlock(&drvdata->spinlock);989 cpus_read_unlock();990 991 /*992 * we only release trace IDs when resetting sysfs.993 * This permits sysfs users to read the trace ID after the trace994 * session has completed. This maintains operational behaviour with995 * prior trace id allocation method996 */997 998 dev_dbg(&csdev->dev, "ETM tracing disabled\n");999}1000 1001static void etm4_disable(struct coresight_device *csdev,1002 struct perf_event *event)1003{1004 enum cs_mode mode;1005 1006 /*1007 * For as long as the tracer isn't disabled another entity can't1008 * change its status. As such we can read the status here without1009 * fearing it will change under us.1010 */1011 mode = coresight_get_mode(csdev);1012 1013 switch (mode) {1014 case CS_MODE_DISABLED:1015 break;1016 case CS_MODE_SYSFS:1017 etm4_disable_sysfs(csdev);1018 break;1019 case CS_MODE_PERF:1020 etm4_disable_perf(csdev, event);1021 break;1022 }1023 1024 if (mode)1025 coresight_set_mode(csdev, CS_MODE_DISABLED);1026}1027 1028static const struct coresight_ops_source etm4_source_ops = {1029 .cpu_id = etm4_cpu_id,1030 .enable = etm4_enable,1031 .disable = etm4_disable,1032};1033 1034static const struct coresight_ops etm4_cs_ops = {1035 .source_ops = &etm4_source_ops,1036};1037 1038static inline bool cpu_supports_sysreg_trace(void)1039{1040 u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);1041 1042 return ((dfr0 >> ID_AA64DFR0_EL1_TraceVer_SHIFT) & 0xfUL) > 0;1043}1044 1045static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata,1046 struct csdev_access *csa)1047{1048 u32 devarch;1049 1050 if (!cpu_supports_sysreg_trace())1051 return false;1052 1053 /*1054 * ETMs implementing sysreg access must implement TRCDEVARCH.1055 */1056 devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH);1057 switch (devarch & ETM_DEVARCH_ID_MASK) {1058 case ETM_DEVARCH_ETMv4x_ARCH:1059 *csa = (struct csdev_access) {1060 .io_mem = false,1061 .read = etm4x_sysreg_read,1062 .write = etm4x_sysreg_write,1063 };1064 break;1065 case ETM_DEVARCH_ETE_ARCH:1066 *csa = (struct csdev_access) {1067 .io_mem = false,1068 .read = ete_sysreg_read,1069 .write = ete_sysreg_write,1070 };1071 break;1072 default:1073 return false;1074 }1075 1076 drvdata->arch = etm_devarch_to_arch(devarch);1077 return true;1078}1079 1080static bool is_devtype_cpu_trace(void __iomem *base)1081{1082 u32 devtype = readl(base + TRCDEVTYPE);1083 1084 return (devtype == CS_DEVTYPE_PE_TRACE);1085}1086 1087static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata,1088 struct csdev_access *csa)1089{1090 u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH);1091 1092 if (!is_coresight_device(drvdata->base) || !is_devtype_cpu_trace(drvdata->base))1093 return false;1094 1095 /*1096 * All ETMs must implement TRCDEVARCH to indicate that1097 * the component is an ETMv4. Even though TRCIDR1 also1098 * contains the information, it is part of the "Trace"1099 * register and must be accessed with the OSLK cleared,1100 * with MMIO. But we cannot touch the OSLK until we are1101 * sure this is an ETM. So rely only on the TRCDEVARCH.1102 */1103 if ((devarch & ETM_DEVARCH_ID_MASK) != ETM_DEVARCH_ETMv4x_ARCH) {1104 pr_warn_once("TRCDEVARCH doesn't match ETMv4 architecture\n");1105 return false;1106 }1107 1108 drvdata->arch = etm_devarch_to_arch(devarch);1109 *csa = CSDEV_ACCESS_IOMEM(drvdata->base);1110 return true;1111}1112 1113static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata,1114 struct csdev_access *csa)1115{1116 /*1117 * Always choose the memory mapped io, if there is1118 * a memory map to prevent sysreg access on broken1119 * systems.1120 */1121 if (drvdata->base)1122 return etm4_init_iomem_access(drvdata, csa);1123 1124 if (etm4_init_sysreg_access(drvdata, csa))1125 return true;1126 1127 return false;1128}1129 1130static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata)1131{1132 u64 dfr0 = read_sysreg(id_aa64dfr0_el1);1133 u64 trfcr;1134 1135 drvdata->trfcr = 0;1136 if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT))1137 return;1138 1139 /*1140 * If the CPU supports v8.4 SelfHosted Tracing, enable1141 * tracing at the kernel EL and EL0, forcing to use the1142 * virtual time as the timestamp.1143 */1144 trfcr = (TRFCR_ELx_TS_VIRTUAL |1145 TRFCR_ELx_ExTRE |1146 TRFCR_ELx_E0TRE);1147 1148 /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */1149 if (is_kernel_in_hyp_mode())1150 trfcr |= TRFCR_EL2_CX;1151 1152 drvdata->trfcr = trfcr;1153}1154 1155/*1156 * The following errata on applicable cpu ranges, affect the CCITMIN filed1157 * in TCRIDR3 register. Software read for the field returns 0x100 limiting1158 * the cycle threshold granularity, whereas the right value should have1159 * been 0x4, which is well supported in the hardware.1160 */1161static struct midr_range etm_wrong_ccitmin_cpus[] = {1162 /* Erratum #1490853 - Cortex-A76 */1163 MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 4, 0),1164 /* Erratum #1490853 - Neoverse-N1 */1165 MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 0, 4, 0),1166 /* Erratum #1491015 - Cortex-A77 */1167 MIDR_RANGE(MIDR_CORTEX_A77, 0, 0, 1, 0),1168 /* Erratum #1502854 - Cortex-X1 */1169 MIDR_REV(MIDR_CORTEX_X1, 0, 0),1170 /* Erratum #1619801 - Neoverse-V1 */1171 MIDR_REV(MIDR_NEOVERSE_V1, 0, 0),1172 {},1173};1174 1175static void etm4_fixup_wrong_ccitmin(struct etmv4_drvdata *drvdata)1176{1177 /*1178 * Erratum affected cpus will read 256 as the minimum1179 * instruction trace cycle counting threshold whereas1180 * the correct value should be 4 instead. Override the1181 * recorded value for 'drvdata->ccitmin' to workaround1182 * this problem.1183 */1184 if (is_midr_in_range_list(read_cpuid_id(), etm_wrong_ccitmin_cpus)) {1185 if (drvdata->ccitmin == 256)1186 drvdata->ccitmin = 4;1187 }1188}1189 1190static void etm4_init_arch_data(void *info)1191{1192 u32 etmidr0;1193 u32 etmidr2;1194 u32 etmidr3;1195 u32 etmidr4;1196 u32 etmidr5;1197 struct etm4_init_arg *init_arg = info;1198 struct etmv4_drvdata *drvdata;1199 struct csdev_access *csa;1200 struct device *dev = init_arg->dev;1201 int i;1202 1203 drvdata = dev_get_drvdata(init_arg->dev);1204 csa = init_arg->csa;1205 1206 /*1207 * If we are unable to detect the access mechanism,1208 * or unable to detect the trace unit type, fail1209 * early.1210 */1211 if (!etm4_init_csdev_access(drvdata, csa))1212 return;1213 1214 if (!csa->io_mem ||1215 fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up"))1216 drvdata->skip_power_up = true;1217 1218 /* Detect the support for OS Lock before we actually use it */1219 etm_detect_os_lock(drvdata, csa);1220 1221 /* Make sure all registers are accessible */1222 etm4_os_unlock_csa(drvdata, csa);1223 etm4_cs_unlock(drvdata, csa);1224 1225 etm4_check_arch_features(drvdata, csa);1226 1227 /* find all capabilities of the tracing unit */1228 etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0);1229 1230 /* INSTP0, bits[2:1] P0 tracing support field */1231 drvdata->instrp0 = !!(FIELD_GET(TRCIDR0_INSTP0_MASK, etmidr0) == 0b11);1232 /* TRCBB, bit[5] Branch broadcast tracing support bit */1233 drvdata->trcbb = !!(etmidr0 & TRCIDR0_TRCBB);1234 /* TRCCOND, bit[6] Conditional instruction tracing support bit */1235 drvdata->trccond = !!(etmidr0 & TRCIDR0_TRCCOND);1236 /* TRCCCI, bit[7] Cycle counting instruction bit */1237 drvdata->trccci = !!(etmidr0 & TRCIDR0_TRCCCI);1238 /* RETSTACK, bit[9] Return stack bit */1239 drvdata->retstack = !!(etmidr0 & TRCIDR0_RETSTACK);1240 /* NUMEVENT, bits[11:10] Number of events field */1241 drvdata->nr_event = FIELD_GET(TRCIDR0_NUMEVENT_MASK, etmidr0);1242 /* QSUPP, bits[16:15] Q element support field */1243 drvdata->q_support = FIELD_GET(TRCIDR0_QSUPP_MASK, etmidr0);1244 if (drvdata->q_support)1245 drvdata->q_filt = !!(etmidr0 & TRCIDR0_QFILT);1246 /* TSSIZE, bits[28:24] Global timestamp size field */1247 drvdata->ts_size = FIELD_GET(TRCIDR0_TSSIZE_MASK, etmidr0);1248 1249 /* maximum size of resources */1250 etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2);1251 /* CIDSIZE, bits[9:5] Indicates the Context ID size */1252 drvdata->ctxid_size = FIELD_GET(TRCIDR2_CIDSIZE_MASK, etmidr2);1253 /* VMIDSIZE, bits[14:10] Indicates the VMID size */1254 drvdata->vmid_size = FIELD_GET(TRCIDR2_VMIDSIZE_MASK, etmidr2);1255 /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */1256 drvdata->ccsize = FIELD_GET(TRCIDR2_CCSIZE_MASK, etmidr2);1257 1258 etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3);1259 /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */1260 drvdata->ccitmin = FIELD_GET(TRCIDR3_CCITMIN_MASK, etmidr3);1261 etm4_fixup_wrong_ccitmin(drvdata);1262 1263 /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */1264 drvdata->s_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_S_MASK, etmidr3);1265 drvdata->config.s_ex_level = drvdata->s_ex_level;1266 /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */1267 drvdata->ns_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_NS_MASK, etmidr3);1268 /*1269 * TRCERR, bit[24] whether a trace unit can trace a1270 * system error exception.1271 */1272 drvdata->trc_error = !!(etmidr3 & TRCIDR3_TRCERR);1273 /* SYNCPR, bit[25] implementation has a fixed synchronization period? */1274 drvdata->syncpr = !!(etmidr3 & TRCIDR3_SYNCPR);1275 /* STALLCTL, bit[26] is stall control implemented? */1276 drvdata->stallctl = !!(etmidr3 & TRCIDR3_STALLCTL);1277 /* SYSSTALL, bit[27] implementation can support stall control? */1278 drvdata->sysstall = !!(etmidr3 & TRCIDR3_SYSSTALL);1279 /*1280 * NUMPROC - the number of PEs available for tracing, 5bits1281 * = TRCIDR3.bits[13:12]bits[30:28]1282 * bits[4:3] = TRCIDR3.bits[13:12] (since etm-v4.2, otherwise RES0)1283 * bits[3:0] = TRCIDR3.bits[30:28]1284 */1285 drvdata->nr_pe = (FIELD_GET(TRCIDR3_NUMPROC_HI_MASK, etmidr3) << 3) |1286 FIELD_GET(TRCIDR3_NUMPROC_LO_MASK, etmidr3);1287 /* NOOVERFLOW, bit[31] is trace overflow prevention supported */1288 drvdata->nooverflow = !!(etmidr3 & TRCIDR3_NOOVERFLOW);1289 1290 /* number of resources trace unit supports */1291 etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4);1292 /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */1293 drvdata->nr_addr_cmp = FIELD_GET(TRCIDR4_NUMACPAIRS_MASK, etmidr4);1294 /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */1295 drvdata->nr_pe_cmp = FIELD_GET(TRCIDR4_NUMPC_MASK, etmidr4);1296 /*1297 * NUMRSPAIR, bits[19:16]1298 * The number of resource pairs conveyed by the HW starts at 0, i.e a1299 * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on.1300 * As such add 1 to the value of NUMRSPAIR for a better representation.1301 *1302 * For ETM v4.3 and later, 0x0 means 0, and no pairs are available -1303 * the default TRUE and FALSE resource selectors are omitted.1304 * Otherwise for values 0x1 and above the number is N + 1 as per v4.2.1305 */1306 drvdata->nr_resource = FIELD_GET(TRCIDR4_NUMRSPAIR_MASK, etmidr4);1307 if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0))1308 drvdata->nr_resource += 1;1309 /*1310 * NUMSSCC, bits[23:20] the number of single-shot1311 * comparator control for tracing. Read any status regs as these1312 * also contain RO capability data.1313 */1314 drvdata->nr_ss_cmp = FIELD_GET(TRCIDR4_NUMSSCC_MASK, etmidr4);1315 for (i = 0; i < drvdata->nr_ss_cmp; i++) {1316 drvdata->config.ss_status[i] =1317 etm4x_relaxed_read32(csa, TRCSSCSRn(i));1318 }1319 /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */1320 drvdata->numcidc = FIELD_GET(TRCIDR4_NUMCIDC_MASK, etmidr4);1321 /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */1322 drvdata->numvmidc = FIELD_GET(TRCIDR4_NUMVMIDC_MASK, etmidr4);1323 1324 etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5);1325 /* NUMEXTIN, bits[8:0] number of external inputs implemented */1326 drvdata->nr_ext_inp = FIELD_GET(TRCIDR5_NUMEXTIN_MASK, etmidr5);1327 /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */1328 drvdata->trcid_size = FIELD_GET(TRCIDR5_TRACEIDSIZE_MASK, etmidr5);1329 /* ATBTRIG, bit[22] implementation can support ATB triggers? */1330 drvdata->atbtrig = !!(etmidr5 & TRCIDR5_ATBTRIG);1331 /*1332 * LPOVERRIDE, bit[23] implementation supports1333 * low-power state override1334 */1335 drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up);1336 /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */1337 drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5);1338 /* NUMCNTR, bits[30:28] number of counters available for tracing */1339 drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5);1340 etm4_cs_lock(drvdata, csa);1341 cpu_detect_trace_filtering(drvdata);1342}1343 1344static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config)1345{1346 return etm4_get_access_type(config) << __bf_shf(TRCVICTLR_EXLEVEL_MASK);1347}1348 1349/* Set ELx trace filter access in the TRCVICTLR register */1350static void etm4_set_victlr_access(struct etmv4_config *config)1351{1352 config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK;1353 config->vinst_ctrl |= etm4_get_victlr_access_type(config);1354}1355 1356static void etm4_set_default_config(struct etmv4_config *config)1357{1358 /* disable all events tracing */1359 config->eventctrl0 = 0x0;1360 config->eventctrl1 = 0x0;1361 1362 /* disable stalling */1363 config->stall_ctrl = 0x0;1364 1365 /* enable trace synchronization every 4096 bytes, if available */1366 config->syncfreq = 0xC;1367 1368 /* disable timestamp event */1369 config->ts_ctrl = 0x0;1370 1371 /* TRCVICTLR::EVENT = 0x01, select the always on logic */1372 config->vinst_ctrl = FIELD_PREP(TRCVICTLR_EVENT_MASK, 0x01);1373 1374 /* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */1375 etm4_set_victlr_access(config);1376}1377 1378static u64 etm4_get_ns_access_type(struct etmv4_config *config)1379{1380 u64 access_type = 0;1381 1382 /*1383 * EXLEVEL_NS, for NonSecure Exception levels.1384 * The mask here is a generic value and must be1385 * shifted to the corresponding field for the registers1386 */1387 if (!is_kernel_in_hyp_mode()) {1388 /* Stay away from hypervisor mode for non-VHE */1389 access_type = ETM_EXLEVEL_NS_HYP;1390 if (config->mode & ETM_MODE_EXCL_KERN)1391 access_type |= ETM_EXLEVEL_NS_OS;1392 } else if (config->mode & ETM_MODE_EXCL_KERN) {1393 access_type = ETM_EXLEVEL_NS_HYP;1394 }1395 1396 if (config->mode & ETM_MODE_EXCL_USER)1397 access_type |= ETM_EXLEVEL_NS_APP;1398 1399 return access_type;1400}1401 1402/*1403 * Construct the exception level masks for a given config.1404 * This must be shifted to the corresponding register field1405 * for usage.1406 */1407static u64 etm4_get_access_type(struct etmv4_config *config)1408{1409 /* All Secure exception levels are excluded from the trace */1410 return etm4_get_ns_access_type(config) | (u64)config->s_ex_level;1411}1412 1413static u64 etm4_get_comparator_access_type(struct etmv4_config *config)1414{1415 return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT;1416}1417 1418static void etm4_set_comparator_filter(struct etmv4_config *config,1419 u64 start, u64 stop, int comparator)1420{1421 u64 access_type = etm4_get_comparator_access_type(config);1422 1423 /* First half of default address comparator */1424 config->addr_val[comparator] = start;1425 config->addr_acc[comparator] = access_type;1426 config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;1427 1428 /* Second half of default address comparator */1429 config->addr_val[comparator + 1] = stop;1430 config->addr_acc[comparator + 1] = access_type;1431 config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;1432 1433 /*1434 * Configure the ViewInst function to include this address range1435 * comparator.1436 *1437 * @comparator is divided by two since it is the index in the1438 * etmv4_config::addr_val array but register TRCVIIECTLR deals with1439 * address range comparator _pairs_.1440 *1441 * Therefore:1442 * index 0 -> compatator pair 01443 * index 2 -> comparator pair 11444 * index 4 -> comparator pair 21445 * ...1446 * index 14 -> comparator pair 71447 */1448 config->viiectlr |= BIT(comparator / 2);1449}1450 1451static void etm4_set_start_stop_filter(struct etmv4_config *config,1452 u64 address, int comparator,1453 enum etm_addr_type type)1454{1455 int shift;1456 u64 access_type = etm4_get_comparator_access_type(config);1457 1458 /* Configure the comparator */1459 config->addr_val[comparator] = address;1460 config->addr_acc[comparator] = access_type;1461 config->addr_type[comparator] = type;1462 1463 /*1464 * Configure ViewInst Start-Stop control register.1465 * Addresses configured to start tracing go from bit 0 to n-1,1466 * while those configured to stop tracing from 16 to 16 + n-1.1467 */1468 shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);1469 config->vissctlr |= BIT(shift + comparator);1470}1471 1472static void etm4_set_default_filter(struct etmv4_config *config)1473{1474 /* Trace everything 'default' filter achieved by no filtering */1475 config->viiectlr = 0x0;1476 1477 /*1478 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is1479 * in the started state1480 */1481 config->vinst_ctrl |= TRCVICTLR_SSSTATUS;1482 config->mode |= ETM_MODE_VIEWINST_STARTSTOP;1483 1484 /* No start-stop filtering for ViewInst */1485 config->vissctlr = 0x0;1486}1487 1488static void etm4_set_default(struct etmv4_config *config)1489{1490 if (WARN_ON_ONCE(!config))1491 return;1492 1493 /*1494 * Make default initialisation trace everything1495 *1496 * This is done by a minimum default config sufficient to enable1497 * full instruction trace - with a default filter for trace all1498 * achieved by having no filtering.1499 */1500 etm4_set_default_config(config);1501 etm4_set_default_filter(config);1502}1503 1504static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)1505{1506 int nr_comparator, index = 0;1507 struct etmv4_config *config = &drvdata->config;1508 1509 /*1510 * nr_addr_cmp holds the number of comparator _pair_, so time 21511 * for the total number of comparators.1512 */1513 nr_comparator = drvdata->nr_addr_cmp * 2;1514 1515 /* Go through the tally of comparators looking for a free one. */1516 while (index < nr_comparator) {1517 switch (type) {1518 case ETM_ADDR_TYPE_RANGE:1519 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&1520 config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)1521 return index;1522 1523 /* Address range comparators go in pairs */1524 index += 2;1525 break;1526 case ETM_ADDR_TYPE_START:1527 case ETM_ADDR_TYPE_STOP:1528 if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)1529 return index;1530 1531 /* Start/stop address can have odd indexes */1532 index += 1;1533 break;1534 default:1535 return -EINVAL;1536 }1537 }1538 1539 /* If we are here all the comparators have been used. */1540 return -ENOSPC;1541}1542 1543static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,1544 struct perf_event *event)1545{1546 int i, comparator, ret = 0;1547 u64 address;1548 struct etmv4_config *config = &drvdata->config;1549 struct etm_filters *filters = event->hw.addr_filters;1550 1551 if (!filters)1552 goto default_filter;1553 1554 /* Sync events with what Perf got */1555 perf_event_addr_filters_sync(event);1556 1557 /*1558 * If there are no filters to deal with simply go ahead with1559 * the default filter, i.e the entire address range.1560 */1561 if (!filters->nr_filters)1562 goto default_filter;1563 1564 for (i = 0; i < filters->nr_filters; i++) {1565 struct etm_filter *filter = &filters->etm_filter[i];1566 enum etm_addr_type type = filter->type;1567 1568 /* See if a comparator is free. */1569 comparator = etm4_get_next_comparator(drvdata, type);1570 if (comparator < 0) {1571 ret = comparator;1572 goto out;1573 }1574 1575 switch (type) {1576 case ETM_ADDR_TYPE_RANGE:1577 etm4_set_comparator_filter(config,1578 filter->start_addr,1579 filter->stop_addr,1580 comparator);1581 /*1582 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is1583 * in the started state1584 */1585 config->vinst_ctrl |= TRCVICTLR_SSSTATUS;1586 1587 /* No start-stop filtering for ViewInst */1588 config->vissctlr = 0x0;1589 break;1590 case ETM_ADDR_TYPE_START:1591 case ETM_ADDR_TYPE_STOP:1592 /* Get the right start or stop address */1593 address = (type == ETM_ADDR_TYPE_START ?1594 filter->start_addr :1595 filter->stop_addr);1596 1597 /* Configure comparator */1598 etm4_set_start_stop_filter(config, address,1599 comparator, type);1600 1601 /*1602 * If filters::ssstatus == 1, trace acquisition was1603 * started but the process was yanked away before the1604 * stop address was hit. As such the start/stop1605 * logic needs to be re-started so that tracing can1606 * resume where it left.1607 *1608 * The start/stop logic status when a process is1609 * scheduled out is checked in function1610 * etm4_disable_perf().1611 */1612 if (filters->ssstatus)1613 config->vinst_ctrl |= TRCVICTLR_SSSTATUS;1614 1615 /* No include/exclude filtering for ViewInst */1616 config->viiectlr = 0x0;1617 break;1618 default:1619 ret = -EINVAL;1620 goto out;1621 }1622 }1623 1624 goto out;1625 1626 1627default_filter:1628 etm4_set_default_filter(config);1629 1630out:1631 return ret;1632}1633 1634void etm4_config_trace_mode(struct etmv4_config *config)1635{1636 u32 mode;1637 1638 mode = config->mode;1639 mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);1640 1641 /* excluding kernel AND user space doesn't make sense */1642 WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));1643 1644 /* nothing to do if neither flags are set */1645 if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))1646 return;1647 1648 etm4_set_victlr_access(config);1649}1650 1651static int etm4_online_cpu(unsigned int cpu)1652{1653 if (!etmdrvdata[cpu])1654 return etm4_probe_cpu(cpu);1655 1656 if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)1657 coresight_enable_sysfs(etmdrvdata[cpu]->csdev);1658 return 0;1659}1660 1661static int etm4_starting_cpu(unsigned int cpu)1662{1663 if (!etmdrvdata[cpu])1664 return 0;1665 1666 spin_lock(&etmdrvdata[cpu]->spinlock);1667 if (!etmdrvdata[cpu]->os_unlock)1668 etm4_os_unlock(etmdrvdata[cpu]);1669 1670 if (coresight_get_mode(etmdrvdata[cpu]->csdev))1671 etm4_enable_hw(etmdrvdata[cpu]);1672 spin_unlock(&etmdrvdata[cpu]->spinlock);1673 return 0;1674}1675 1676static int etm4_dying_cpu(unsigned int cpu)1677{1678 if (!etmdrvdata[cpu])1679 return 0;1680 1681 spin_lock(&etmdrvdata[cpu]->spinlock);1682 if (coresight_get_mode(etmdrvdata[cpu]->csdev))1683 etm4_disable_hw(etmdrvdata[cpu]);1684 spin_unlock(&etmdrvdata[cpu]->spinlock);1685 return 0;1686}1687 1688static int __etm4_cpu_save(struct etmv4_drvdata *drvdata)1689{1690 int i, ret = 0;1691 struct etmv4_save_state *state;1692 struct coresight_device *csdev = drvdata->csdev;1693 struct csdev_access *csa;1694 struct device *etm_dev;1695 1696 if (WARN_ON(!csdev))1697 return -ENODEV;1698 1699 etm_dev = &csdev->dev;1700 csa = &csdev->access;1701 1702 /*1703 * As recommended by 3.4.1 ("The procedure when powering down the PE")1704 * of ARM IHI 0064D1705 */1706 dsb(sy);1707 isb();1708 1709 etm4_cs_unlock(drvdata, csa);1710 /* Lock the OS lock to disable trace and external debugger access */1711 etm4_os_lock(drvdata);1712 1713 /* wait for TRCSTATR.PMSTABLE to go up */1714 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) {1715 dev_err(etm_dev,1716 "timeout while waiting for PM Stable Status\n");1717 etm4_os_unlock(drvdata);1718 ret = -EBUSY;1719 goto out;1720 }1721 1722 state = drvdata->save_state;1723 1724 state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR);1725 if (drvdata->nr_pe)1726 state->trcprocselr = etm4x_read32(csa, TRCPROCSELR);1727 state->trcconfigr = etm4x_read32(csa, TRCCONFIGR);1728 state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR);1729 state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R);1730 state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R);1731 if (drvdata->stallctl)1732 state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR);1733 state->trctsctlr = etm4x_read32(csa, TRCTSCTLR);1734 state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR);1735 state->trcccctlr = etm4x_read32(csa, TRCCCCTLR);1736 state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR);1737 state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR);1738 if (drvdata->q_filt)1739 state->trcqctlr = etm4x_read32(csa, TRCQCTLR);1740 1741 state->trcvictlr = etm4x_read32(csa, TRCVICTLR);1742 state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR);1743 state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR);1744 if (drvdata->nr_pe_cmp)1745 state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR);1746 1747 for (i = 0; i < drvdata->nrseqstate - 1; i++)1748 state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i));1749 1750 if (drvdata->nrseqstate) {1751 state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR);1752 state->trcseqstr = etm4x_read32(csa, TRCSEQSTR);1753 }1754 state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR);1755 1756 for (i = 0; i < drvdata->nr_cntr; i++) {1757 state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i));1758 state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i));1759 state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i));1760 }1761 1762 /* Resource selector pair 0 is reserved */1763 for (i = 2; i < drvdata->nr_resource * 2; i++)1764 state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i));1765 1766 for (i = 0; i < drvdata->nr_ss_cmp; i++) {1767 state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i));1768 state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i));1769 if (etm4x_sspcicrn_present(drvdata, i))1770 state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i));1771 }1772 1773 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {1774 state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i));1775 state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i));1776 }1777 1778 /*1779 * Data trace stream is architecturally prohibited for A profile cores1780 * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per1781 * section 1.3.4 ("Possible functional configurations of an ETMv4 trace1782 * unit") of ARM IHI 0064D.1783 */1784 1785 for (i = 0; i < drvdata->numcidc; i++)1786 state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i));1787 1788 for (i = 0; i < drvdata->numvmidc; i++)1789 state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i));1790 1791 state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0);1792 if (drvdata->numcidc > 4)1793 state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1);1794 1795 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0);1796 if (drvdata->numvmidc > 4)1797 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1);1798 1799 state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR);1800 1801 if (!drvdata->skip_power_up)1802 state->trcpdcr = etm4x_read32(csa, TRCPDCR);1803 1804 /* wait for TRCSTATR.IDLE to go up */1805 if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) {1806 dev_err(etm_dev,1807 "timeout while waiting for Idle Trace Status\n");1808 etm4_os_unlock(drvdata);1809 ret = -EBUSY;1810 goto out;1811 }1812 1813 drvdata->state_needs_restore = true;1814 1815 /*1816 * Power can be removed from the trace unit now. We do this to1817 * potentially save power on systems that respect the TRCPDCR_PU1818 * despite requesting software to save/restore state.1819 */1820 if (!drvdata->skip_power_up)1821 etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU),1822 TRCPDCR);1823out:1824 etm4_cs_lock(drvdata, csa);1825 return ret;1826}1827 1828static int etm4_cpu_save(struct etmv4_drvdata *drvdata)1829{1830 int ret = 0;1831 1832 /* Save the TRFCR irrespective of whether the ETM is ON */1833 if (drvdata->trfcr)1834 drvdata->save_trfcr = read_trfcr();1835 /*1836 * Save and restore the ETM Trace registers only if1837 * the ETM is active.1838 */1839 if (coresight_get_mode(drvdata->csdev) && drvdata->save_state)1840 ret = __etm4_cpu_save(drvdata);1841 return ret;1842}1843 1844static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)1845{1846 int i;1847 struct etmv4_save_state *state = drvdata->save_state;1848 struct csdev_access *csa = &drvdata->csdev->access;1849 1850 if (WARN_ON(!drvdata->csdev))1851 return;1852 1853 etm4_cs_unlock(drvdata, csa);1854 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);1855 1856 etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR);1857 if (drvdata->nr_pe)1858 etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR);1859 etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR);1860 etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR);1861 etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R);1862 etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R);1863 if (drvdata->stallctl)1864 etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR);1865 etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR);1866 etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR);1867 etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR);1868 etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR);1869 etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR);1870 if (drvdata->q_filt)1871 etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR);1872 1873 etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR);1874 etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR);1875 etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR);1876 if (drvdata->nr_pe_cmp)1877 etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR);1878 1879 for (i = 0; i < drvdata->nrseqstate - 1; i++)1880 etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i));1881 1882 if (drvdata->nrseqstate) {1883 etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR);1884 etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR);1885 }1886 etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR);1887 1888 for (i = 0; i < drvdata->nr_cntr; i++) {1889 etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i));1890 etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i));1891 etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i));1892 }1893 1894 /* Resource selector pair 0 is reserved */1895 for (i = 2; i < drvdata->nr_resource * 2; i++)1896 etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i));1897 1898 for (i = 0; i < drvdata->nr_ss_cmp; i++) {1899 etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i));1900 etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i));1901 if (etm4x_sspcicrn_present(drvdata, i))1902 etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i));1903 }1904 1905 for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {1906 etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i));1907 etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i));1908 }1909 1910 for (i = 0; i < drvdata->numcidc; i++)1911 etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i));1912 1913 for (i = 0; i < drvdata->numvmidc; i++)1914 etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i));1915 1916 etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0);1917 if (drvdata->numcidc > 4)1918 etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1);1919 1920 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0);1921 if (drvdata->numvmidc > 4)1922 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1);1923 1924 etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);1925 1926 if (!drvdata->skip_power_up)1927 etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR);1928 1929 drvdata->state_needs_restore = false;1930 1931 /*1932 * As recommended by section 4.3.7 ("Synchronization when using the1933 * memory-mapped interface") of ARM IHI 0064D1934 */1935 dsb(sy);1936 isb();1937 1938 /* Unlock the OS lock to re-enable trace and external debug access */1939 etm4_os_unlock(drvdata);1940 etm4_cs_lock(drvdata, csa);1941}1942 1943static void etm4_cpu_restore(struct etmv4_drvdata *drvdata)1944{1945 if (drvdata->trfcr)1946 write_trfcr(drvdata->save_trfcr);1947 if (drvdata->state_needs_restore)1948 __etm4_cpu_restore(drvdata);1949}1950 1951static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,1952 void *v)1953{1954 struct etmv4_drvdata *drvdata;1955 unsigned int cpu = smp_processor_id();1956 1957 if (!etmdrvdata[cpu])1958 return NOTIFY_OK;1959 1960 drvdata = etmdrvdata[cpu];1961 1962 if (WARN_ON_ONCE(drvdata->cpu != cpu))1963 return NOTIFY_BAD;1964 1965 switch (cmd) {1966 case CPU_PM_ENTER:1967 if (etm4_cpu_save(drvdata))1968 return NOTIFY_BAD;1969 break;1970 case CPU_PM_EXIT:1971 case CPU_PM_ENTER_FAILED:1972 etm4_cpu_restore(drvdata);1973 break;1974 default:1975 return NOTIFY_DONE;1976 }1977 1978 return NOTIFY_OK;1979}1980 1981static struct notifier_block etm4_cpu_pm_nb = {1982 .notifier_call = etm4_cpu_pm_notify,1983};1984 1985/* Setup PM. Deals with error conditions and counts */1986static int __init etm4_pm_setup(void)1987{1988 int ret;1989 1990 ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb);1991 if (ret)1992 return ret;1993 1994 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,1995 "arm/coresight4:starting",1996 etm4_starting_cpu, etm4_dying_cpu);1997 1998 if (ret)1999 goto unregister_notifier;2000 2001 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,2002 "arm/coresight4:online",2003 etm4_online_cpu, NULL);2004 2005 /* HP dyn state ID returned in ret on success */2006 if (ret > 0) {2007 hp_online = ret;2008 return 0;2009 }2010 2011 /* failed dyn state - remove others */2012 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);2013 2014unregister_notifier:2015 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);2016 return ret;2017}2018 2019static void etm4_pm_clear(void)2020{2021 cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);2022 cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);2023 if (hp_online) {2024 cpuhp_remove_state_nocalls(hp_online);2025 hp_online = 0;2026 }2027}2028 2029static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)2030{2031 int ret;2032 struct coresight_platform_data *pdata = NULL;2033 struct device *dev = init_arg->dev;2034 struct etmv4_drvdata *drvdata = dev_get_drvdata(dev);2035 struct coresight_desc desc = { 0 };2036 u8 major, minor;2037 char *type_name;2038 2039 if (!drvdata)2040 return -EINVAL;2041 2042 desc.access = *init_arg->csa;2043 2044 if (!drvdata->arch)2045 return -EINVAL;2046 2047 major = ETM_ARCH_MAJOR_VERSION(drvdata->arch);2048 minor = ETM_ARCH_MINOR_VERSION(drvdata->arch);2049 2050 if (etm4x_is_ete(drvdata)) {2051 type_name = "ete";2052 /* ETE v1 has major version == 0b101. Adjust this for logging.*/2053 major -= 4;2054 } else {2055 type_name = "etm";2056 }2057 2058 desc.name = devm_kasprintf(dev, GFP_KERNEL,2059 "%s%d", type_name, drvdata->cpu);2060 if (!desc.name)2061 return -ENOMEM;2062 2063 etm4_set_default(&drvdata->config);2064 2065 pdata = coresight_get_platform_data(dev);2066 if (IS_ERR(pdata))2067 return PTR_ERR(pdata);2068 2069 dev->platform_data = pdata;2070 2071 desc.type = CORESIGHT_DEV_TYPE_SOURCE;2072 desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;2073 desc.ops = &etm4_cs_ops;2074 desc.pdata = pdata;2075 desc.dev = dev;2076 desc.groups = coresight_etmv4_groups;2077 drvdata->csdev = coresight_register(&desc);2078 if (IS_ERR(drvdata->csdev))2079 return PTR_ERR(drvdata->csdev);2080 2081 ret = etm_perf_symlink(drvdata->csdev, true);2082 if (ret) {2083 coresight_unregister(drvdata->csdev);2084 return ret;2085 }2086 2087 /* register with config infrastructure & load any current features */2088 ret = etm4_cscfg_register(drvdata->csdev);2089 if (ret) {2090 coresight_unregister(drvdata->csdev);2091 return ret;2092 }2093 2094 etmdrvdata[drvdata->cpu] = drvdata;2095 2096 dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n",2097 drvdata->cpu, type_name, major, minor);2098 2099 if (boot_enable) {2100 coresight_enable_sysfs(drvdata->csdev);2101 drvdata->boot_enable = true;2102 }2103 2104 return 0;2105}2106 2107static int etm4_probe(struct device *dev)2108{2109 struct etmv4_drvdata *drvdata = dev_get_drvdata(dev);2110 struct csdev_access access = { 0 };2111 struct etm4_init_arg init_arg = { 0 };2112 struct etm4_init_arg *delayed;2113 2114 if (WARN_ON(!drvdata))2115 return -ENOMEM;2116 2117 if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)2118 pm_save_enable = coresight_loses_context_with_cpu(dev) ?2119 PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER;2120 2121 if (pm_save_enable != PARAM_PM_SAVE_NEVER) {2122 drvdata->save_state = devm_kmalloc(dev,2123 sizeof(struct etmv4_save_state), GFP_KERNEL);2124 if (!drvdata->save_state)2125 return -ENOMEM;2126 }2127 2128 spin_lock_init(&drvdata->spinlock);2129 2130 drvdata->cpu = coresight_get_cpu(dev);2131 if (drvdata->cpu < 0)2132 return drvdata->cpu;2133 2134 init_arg.dev = dev;2135 init_arg.csa = &access;2136 2137 /*2138 * Serialize against CPUHP callbacks to avoid race condition2139 * between the smp call and saving the delayed probe.2140 */2141 cpus_read_lock();2142 if (smp_call_function_single(drvdata->cpu,2143 etm4_init_arch_data, &init_arg, 1)) {2144 /* The CPU was offline, try again once it comes online. */2145 delayed = devm_kmalloc(dev, sizeof(*delayed), GFP_KERNEL);2146 if (!delayed) {2147 cpus_read_unlock();2148 return -ENOMEM;2149 }2150 2151 *delayed = init_arg;2152 2153 per_cpu(delayed_probe, drvdata->cpu) = delayed;2154 2155 cpus_read_unlock();2156 return 0;2157 }2158 cpus_read_unlock();2159 2160 return etm4_add_coresight_dev(&init_arg);2161}2162 2163static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id)2164{2165 struct etmv4_drvdata *drvdata;2166 void __iomem *base;2167 struct device *dev = &adev->dev;2168 struct resource *res = &adev->res;2169 int ret;2170 2171 /* Validity for the resource is already checked by the AMBA core */2172 base = devm_ioremap_resource(dev, res);2173 if (IS_ERR(base))2174 return PTR_ERR(base);2175 2176 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);2177 if (!drvdata)2178 return -ENOMEM;2179 2180 drvdata->base = base;2181 dev_set_drvdata(dev, drvdata);2182 ret = etm4_probe(dev);2183 if (!ret)2184 pm_runtime_put(&adev->dev);2185 2186 return ret;2187}2188 2189static int etm4_probe_platform_dev(struct platform_device *pdev)2190{2191 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);2192 struct etmv4_drvdata *drvdata;2193 int ret;2194 2195 drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL);2196 if (!drvdata)2197 return -ENOMEM;2198 2199 drvdata->pclk = coresight_get_enable_apb_pclk(&pdev->dev);2200 if (IS_ERR(drvdata->pclk))2201 return -ENODEV;2202 2203 if (res) {2204 drvdata->base = devm_ioremap_resource(&pdev->dev, res);2205 if (IS_ERR(drvdata->base)) {2206 clk_put(drvdata->pclk);2207 return PTR_ERR(drvdata->base);2208 }2209 }2210 2211 dev_set_drvdata(&pdev->dev, drvdata);2212 pm_runtime_get_noresume(&pdev->dev);2213 pm_runtime_set_active(&pdev->dev);2214 pm_runtime_enable(&pdev->dev);2215 2216 ret = etm4_probe(&pdev->dev);2217 2218 pm_runtime_put(&pdev->dev);2219 if (ret)2220 pm_runtime_disable(&pdev->dev);2221 2222 return ret;2223}2224 2225static int etm4_probe_cpu(unsigned int cpu)2226{2227 int ret;2228 struct etm4_init_arg init_arg;2229 struct csdev_access access = { 0 };2230 struct etm4_init_arg *iap = *this_cpu_ptr(&delayed_probe);2231 2232 if (!iap)2233 return 0;2234 2235 init_arg = *iap;2236 devm_kfree(init_arg.dev, iap);2237 *this_cpu_ptr(&delayed_probe) = NULL;2238 2239 ret = pm_runtime_resume_and_get(init_arg.dev);2240 if (ret < 0) {2241 dev_err(init_arg.dev, "Failed to get PM runtime!\n");2242 return 0;2243 }2244 2245 init_arg.csa = &access;2246 etm4_init_arch_data(&init_arg);2247 2248 etm4_add_coresight_dev(&init_arg);2249 2250 pm_runtime_put(init_arg.dev);2251 return 0;2252}2253 2254static struct amba_cs_uci_id uci_id_etm4[] = {2255 {2256 /* ETMv4 UCI data */2257 .devarch = ETM_DEVARCH_ETMv4x_ARCH,2258 .devarch_mask = ETM_DEVARCH_ID_MASK,2259 .devtype = CS_DEVTYPE_PE_TRACE,2260 }2261};2262 2263static void clear_etmdrvdata(void *info)2264{2265 int cpu = *(int *)info;2266 2267 etmdrvdata[cpu] = NULL;2268 per_cpu(delayed_probe, cpu) = NULL;2269}2270 2271static void etm4_remove_dev(struct etmv4_drvdata *drvdata)2272{2273 bool had_delayed_probe;2274 /*2275 * Taking hotplug lock here to avoid racing between etm4_remove_dev()2276 * and CPU hotplug call backs.2277 */2278 cpus_read_lock();2279 2280 had_delayed_probe = per_cpu(delayed_probe, drvdata->cpu);2281 2282 /*2283 * The readers for etmdrvdata[] are CPU hotplug call backs2284 * and PM notification call backs. Change etmdrvdata[i] on2285 * CPU i ensures these call backs has consistent view2286 * inside one call back function.2287 */2288 if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1))2289 clear_etmdrvdata(&drvdata->cpu);2290 2291 cpus_read_unlock();2292 2293 if (!had_delayed_probe) {2294 etm_perf_symlink(drvdata->csdev, false);2295 cscfg_unregister_csdev(drvdata->csdev);2296 coresight_unregister(drvdata->csdev);2297 }2298}2299 2300static void etm4_remove_amba(struct amba_device *adev)2301{2302 struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev);2303 2304 if (drvdata)2305 etm4_remove_dev(drvdata);2306}2307 2308static void etm4_remove_platform_dev(struct platform_device *pdev)2309{2310 struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev);2311 2312 if (drvdata)2313 etm4_remove_dev(drvdata);2314 pm_runtime_disable(&pdev->dev);2315 2316 if (drvdata && !IS_ERR_OR_NULL(drvdata->pclk))2317 clk_put(drvdata->pclk);2318}2319 2320static const struct amba_id etm4_ids[] = {2321 CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */2322 CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */2323 CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */2324 CS_AMBA_ID(0x000bb959), /* Cortex-A73 */2325 CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */2326 CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */2327 CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */2328 CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */2329 CS_AMBA_UCI_ID(0x000bbd41, uci_id_etm4),/* Cortex-A78 */2330 CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */2331 CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */2332 CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */2333 CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */2334 CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */2335 CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */2336 CS_AMBA_UCI_ID(0x000bbd0d, uci_id_etm4),/* Qualcomm Kryo 5XX Cortex-A77 */2337 CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */2338 CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */2339 CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */2340 /*2341 * Match all PIDs with ETM4 DEVARCH. No need for adding any of the new2342 * CPUs to the list here.2343 */2344 CS_AMBA_MATCH_ALL_UCI(uci_id_etm4),2345 {},2346};2347 2348MODULE_DEVICE_TABLE(amba, etm4_ids);2349 2350static struct amba_driver etm4x_amba_driver = {2351 .drv = {2352 .name = "coresight-etm4x",2353 .suppress_bind_attrs = true,2354 },2355 .probe = etm4_probe_amba,2356 .remove = etm4_remove_amba,2357 .id_table = etm4_ids,2358};2359 2360#ifdef CONFIG_PM2361static int etm4_runtime_suspend(struct device *dev)2362{2363 struct etmv4_drvdata *drvdata = dev_get_drvdata(dev);2364 2365 if (drvdata->pclk && !IS_ERR(drvdata->pclk))2366 clk_disable_unprepare(drvdata->pclk);2367 2368 return 0;2369}2370 2371static int etm4_runtime_resume(struct device *dev)2372{2373 struct etmv4_drvdata *drvdata = dev_get_drvdata(dev);2374 2375 if (drvdata->pclk && !IS_ERR(drvdata->pclk))2376 clk_prepare_enable(drvdata->pclk);2377 2378 return 0;2379}2380#endif2381 2382static const struct dev_pm_ops etm4_dev_pm_ops = {2383 SET_RUNTIME_PM_OPS(etm4_runtime_suspend, etm4_runtime_resume, NULL)2384};2385 2386static const struct of_device_id etm4_sysreg_match[] = {2387 { .compatible = "arm,coresight-etm4x-sysreg" },2388 { .compatible = "arm,embedded-trace-extension" },2389 {}2390};2391 2392#ifdef CONFIG_ACPI2393static const struct acpi_device_id etm4x_acpi_ids[] = {2394 {"ARMHC500", 0, 0, 0}, /* ARM CoreSight ETM4x */2395 {}2396};2397MODULE_DEVICE_TABLE(acpi, etm4x_acpi_ids);2398#endif2399 2400static struct platform_driver etm4_platform_driver = {2401 .probe = etm4_probe_platform_dev,2402 .remove_new = etm4_remove_platform_dev,2403 .driver = {2404 .name = "coresight-etm4x",2405 .of_match_table = etm4_sysreg_match,2406 .acpi_match_table = ACPI_PTR(etm4x_acpi_ids),2407 .suppress_bind_attrs = true,2408 .pm = &etm4_dev_pm_ops,2409 },2410};2411 2412static int __init etm4x_init(void)2413{2414 int ret;2415 2416 ret = etm4_pm_setup();2417 2418 /* etm4_pm_setup() does its own cleanup - exit on error */2419 if (ret)2420 return ret;2421 2422 ret = amba_driver_register(&etm4x_amba_driver);2423 if (ret) {2424 pr_err("Error registering etm4x AMBA driver\n");2425 goto clear_pm;2426 }2427 2428 ret = platform_driver_register(&etm4_platform_driver);2429 if (!ret)2430 return 0;2431 2432 pr_err("Error registering etm4x platform driver\n");2433 amba_driver_unregister(&etm4x_amba_driver);2434 2435clear_pm:2436 etm4_pm_clear();2437 return ret;2438}2439 2440static void __exit etm4x_exit(void)2441{2442 amba_driver_unregister(&etm4x_amba_driver);2443 platform_driver_unregister(&etm4_platform_driver);2444 etm4_pm_clear();2445}2446 2447module_init(etm4x_init);2448module_exit(etm4x_exit);2449 2450MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>");2451MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>");2452MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver");2453MODULE_LICENSE("GPL v2");2454