1976 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.4 *5 * ARMv7 support: Jean Pihet <jpihet@mvista.com>6 * 2010 (c) MontaVista Software, LLC.7 *8 * Copied from ARMv6 code, with the low level code inspired9 * by the ARMv7 Oprofile code.10 *11 * Cortex-A8 has up to 4 configurable performance counters and12 * a single cycle counter.13 * Cortex-A9 has up to 31 configurable performance counters and14 * a single cycle counter.15 *16 * All counters can be enabled/disabled and IRQ masked separately. The cycle17 * counter and all 4 performance counters together can be reset separately.18 */19 20#include <asm/cp15.h>21#include <asm/cputype.h>22#include <asm/irq_regs.h>23#include <asm/vfp.h>24#include "../vfp/vfpinstr.h"25 26#include <linux/of.h>27#include <linux/perf/arm_pmu.h>28#include <linux/platform_device.h>29 30/*31 * Common ARMv7 event types32 *33 * Note: An implementation may not be able to count all of these events34 * but the encodings are considered to be `reserved' in the case that35 * they are not available.36 */37#define ARMV7_PERFCTR_PMNC_SW_INCR 0x0038#define ARMV7_PERFCTR_L1_ICACHE_REFILL 0x0139#define ARMV7_PERFCTR_ITLB_REFILL 0x0240#define ARMV7_PERFCTR_L1_DCACHE_REFILL 0x0341#define ARMV7_PERFCTR_L1_DCACHE_ACCESS 0x0442#define ARMV7_PERFCTR_DTLB_REFILL 0x0543#define ARMV7_PERFCTR_MEM_READ 0x0644#define ARMV7_PERFCTR_MEM_WRITE 0x0745#define ARMV7_PERFCTR_INSTR_EXECUTED 0x0846#define ARMV7_PERFCTR_EXC_TAKEN 0x0947#define ARMV7_PERFCTR_EXC_EXECUTED 0x0A48#define ARMV7_PERFCTR_CID_WRITE 0x0B49 50/*51 * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.52 * It counts:53 * - all (taken) branch instructions,54 * - instructions that explicitly write the PC,55 * - exception generating instructions.56 */57#define ARMV7_PERFCTR_PC_WRITE 0x0C58#define ARMV7_PERFCTR_PC_IMM_BRANCH 0x0D59#define ARMV7_PERFCTR_PC_PROC_RETURN 0x0E60#define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS 0x0F61#define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED 0x1062#define ARMV7_PERFCTR_CLOCK_CYCLES 0x1163#define ARMV7_PERFCTR_PC_BRANCH_PRED 0x1264 65/* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */66#define ARMV7_PERFCTR_MEM_ACCESS 0x1367#define ARMV7_PERFCTR_L1_ICACHE_ACCESS 0x1468#define ARMV7_PERFCTR_L1_DCACHE_WB 0x1569#define ARMV7_PERFCTR_L2_CACHE_ACCESS 0x1670#define ARMV7_PERFCTR_L2_CACHE_REFILL 0x1771#define ARMV7_PERFCTR_L2_CACHE_WB 0x1872#define ARMV7_PERFCTR_BUS_ACCESS 0x1973#define ARMV7_PERFCTR_MEM_ERROR 0x1A74#define ARMV7_PERFCTR_INSTR_SPEC 0x1B75#define ARMV7_PERFCTR_TTBR_WRITE 0x1C76#define ARMV7_PERFCTR_BUS_CYCLES 0x1D77 78#define ARMV7_PERFCTR_CPU_CYCLES 0xFF79 80/* ARMv7 Cortex-A8 specific event types */81#define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS 0x4382#define ARMV7_A8_PERFCTR_L2_CACHE_REFILL 0x4483#define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS 0x5084#define ARMV7_A8_PERFCTR_STALL_ISIDE 0x5685 86/* ARMv7 Cortex-A9 specific event types */87#define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME 0x6888#define ARMV7_A9_PERFCTR_STALL_ICACHE 0x6089#define ARMV7_A9_PERFCTR_STALL_DISPATCH 0x6690 91/* ARMv7 Cortex-A5 specific event types */92#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL 0xc293#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP 0xc394 95/* ARMv7 Cortex-A15 specific event types */96#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ 0x4097#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x4198#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ 0x4299#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE 0x43100 101#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ 0x4C102#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE 0x4D103 104#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ 0x50105#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51106#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ 0x52107#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE 0x53108 109#define ARMV7_A15_PERFCTR_PC_WRITE_SPEC 0x76110 111/* ARMv7 Cortex-A12 specific event types */112#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ 0x40113#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41114 115#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ 0x50116#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51117 118#define ARMV7_A12_PERFCTR_PC_WRITE_SPEC 0x76119 120#define ARMV7_A12_PERFCTR_PF_TLB_REFILL 0xe7121 122/* ARMv7 Krait specific event types */123#define KRAIT_PMRESR0_GROUP0 0xcc124#define KRAIT_PMRESR1_GROUP0 0xd0125#define KRAIT_PMRESR2_GROUP0 0xd4126#define KRAIT_VPMRESR0_GROUP0 0xd8127 128#define KRAIT_PERFCTR_L1_ICACHE_ACCESS 0x10011129#define KRAIT_PERFCTR_L1_ICACHE_MISS 0x10010130 131#define KRAIT_PERFCTR_L1_ITLB_ACCESS 0x12222132#define KRAIT_PERFCTR_L1_DTLB_ACCESS 0x12210133 134/* ARMv7 Scorpion specific event types */135#define SCORPION_LPM0_GROUP0 0x4c136#define SCORPION_LPM1_GROUP0 0x50137#define SCORPION_LPM2_GROUP0 0x54138#define SCORPION_L2LPM_GROUP0 0x58139#define SCORPION_VLPM_GROUP0 0x5c140 141#define SCORPION_ICACHE_ACCESS 0x10053142#define SCORPION_ICACHE_MISS 0x10052143 144#define SCORPION_DTLB_ACCESS 0x12013145#define SCORPION_DTLB_MISS 0x12012146 147#define SCORPION_ITLB_MISS 0x12021148 149/*150 * Cortex-A8 HW events mapping151 *152 * The hardware events that we support. We do support cache operations but153 * we have harvard caches and no way to combine instruction and data154 * accesses/misses in hardware.155 */156static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {157 PERF_MAP_ALL_UNSUPPORTED,158 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,159 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,160 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,161 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,162 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,163 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,164 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,165};166 167static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]168 [PERF_COUNT_HW_CACHE_OP_MAX]169 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {170 PERF_CACHE_MAP_ALL_UNSUPPORTED,171 172 /*173 * The performance counters don't differentiate between read and write174 * accesses/misses so this isn't strictly correct, but it's the best we175 * can do. Writes and reads get combined.176 */177 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,178 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,179 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,180 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,181 182 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,183 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,184 185 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,186 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,187 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,188 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,189 190 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,191 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,192 193 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,194 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,195 196 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,197 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,198 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,199 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,200};201 202/*203 * Cortex-A9 HW events mapping204 */205static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {206 PERF_MAP_ALL_UNSUPPORTED,207 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,208 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,209 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,210 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,211 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,212 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,213 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,214 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH,215};216 217static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]218 [PERF_COUNT_HW_CACHE_OP_MAX]219 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {220 PERF_CACHE_MAP_ALL_UNSUPPORTED,221 222 /*223 * The performance counters don't differentiate between read and write224 * accesses/misses so this isn't strictly correct, but it's the best we225 * can do. Writes and reads get combined.226 */227 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,228 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,229 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,230 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,231 232 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,233 234 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,235 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,236 237 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,238 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,239 240 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,241 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,242 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,243 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,244};245 246/*247 * Cortex-A5 HW events mapping248 */249static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {250 PERF_MAP_ALL_UNSUPPORTED,251 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,252 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,253 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,254 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,255 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,256 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,257};258 259static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]260 [PERF_COUNT_HW_CACHE_OP_MAX]261 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {262 PERF_CACHE_MAP_ALL_UNSUPPORTED,263 264 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,265 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,266 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,267 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,268 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,269 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,270 271 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,272 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,273 /*274 * The prefetch counters don't differentiate between the I side and the275 * D side.276 */277 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,278 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,279 280 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,281 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,282 283 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,284 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,285 286 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,287 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,288 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,289 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,290};291 292/*293 * Cortex-A15 HW events mapping294 */295static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {296 PERF_MAP_ALL_UNSUPPORTED,297 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,298 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,299 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,300 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,301 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,302 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,303 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,304};305 306static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]307 [PERF_COUNT_HW_CACHE_OP_MAX]308 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {309 PERF_CACHE_MAP_ALL_UNSUPPORTED,310 311 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,312 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,313 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,314 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,315 316 /*317 * Not all performance counters differentiate between read and write318 * accesses/misses so we're not always strictly correct, but it's the319 * best we can do. Writes and reads get combined in these cases.320 */321 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,322 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,323 324 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,325 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,326 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,327 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,328 329 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,330 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,331 332 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,333 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,334 335 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,336 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,337 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,338 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,339};340 341/*342 * Cortex-A7 HW events mapping343 */344static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {345 PERF_MAP_ALL_UNSUPPORTED,346 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,347 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,348 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,349 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,350 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,351 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,352 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,353};354 355static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]356 [PERF_COUNT_HW_CACHE_OP_MAX]357 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {358 PERF_CACHE_MAP_ALL_UNSUPPORTED,359 360 /*361 * The performance counters don't differentiate between read and write362 * accesses/misses so this isn't strictly correct, but it's the best we363 * can do. Writes and reads get combined.364 */365 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,366 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,367 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,368 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,369 370 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,371 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,372 373 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,374 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,375 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,376 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,377 378 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,379 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,380 381 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,382 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,383 384 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,385 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,386 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,387 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,388};389 390/*391 * Cortex-A12 HW events mapping392 */393static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {394 PERF_MAP_ALL_UNSUPPORTED,395 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,396 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,397 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,398 [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,399 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,400 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,401 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,402};403 404static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]405 [PERF_COUNT_HW_CACHE_OP_MAX]406 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {407 PERF_CACHE_MAP_ALL_UNSUPPORTED,408 409 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,410 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,411 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,412 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,413 414 /*415 * Not all performance counters differentiate between read and write416 * accesses/misses so we're not always strictly correct, but it's the417 * best we can do. Writes and reads get combined in these cases.418 */419 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,420 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,421 422 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,423 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,424 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,425 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,426 427 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,428 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,429 [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL,430 431 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,432 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,433 434 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,435 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,436 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,437 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,438};439 440/*441 * Krait HW events mapping442 */443static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {444 PERF_MAP_ALL_UNSUPPORTED,445 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,446 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,447 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,448 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,449 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,450};451 452static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {453 PERF_MAP_ALL_UNSUPPORTED,454 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,455 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,456 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,457 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,458};459 460static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]461 [PERF_COUNT_HW_CACHE_OP_MAX]462 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {463 PERF_CACHE_MAP_ALL_UNSUPPORTED,464 465 /*466 * The performance counters don't differentiate between read and write467 * accesses/misses so this isn't strictly correct, but it's the best we468 * can do. Writes and reads get combined.469 */470 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,471 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,472 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,473 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,474 475 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS,476 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS,477 478 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,479 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,480 481 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,482 [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,483 484 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,485 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,486 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,487 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,488};489 490/*491 * Scorpion HW events mapping492 */493static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {494 PERF_MAP_ALL_UNSUPPORTED,495 [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,496 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,497 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,498 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,499 [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES,500};501 502static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]503 [PERF_COUNT_HW_CACHE_OP_MAX]504 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {505 PERF_CACHE_MAP_ALL_UNSUPPORTED,506 /*507 * The performance counters don't differentiate between read and write508 * accesses/misses so this isn't strictly correct, but it's the best we509 * can do. Writes and reads get combined.510 */511 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,512 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,513 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,514 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,515 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,516 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,517 /*518 * Only ITLB misses and DTLB refills are supported. If users want the519 * DTLB refills misses a raw counter must be used.520 */521 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,522 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,523 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,524 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,525 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,526 [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,527 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,528 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,529 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,530 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,531};532 533PMU_FORMAT_ATTR(event, "config:0-7");534 535static struct attribute *armv7_pmu_format_attrs[] = {536 &format_attr_event.attr,537 NULL,538};539 540static struct attribute_group armv7_pmu_format_attr_group = {541 .name = "format",542 .attrs = armv7_pmu_format_attrs,543};544 545#define ARMV7_EVENT_ATTR_RESOLVE(m) #m546#define ARMV7_EVENT_ATTR(name, config) \547 PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \548 "event=" ARMV7_EVENT_ATTR_RESOLVE(config))549 550ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR);551ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL);552ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL);553ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL);554ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS);555ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL);556ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ);557ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE);558ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED);559ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN);560ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED);561ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE);562ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE);563ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH);564ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN);565ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS);566ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED);567ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES);568ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED);569 570static struct attribute *armv7_pmuv1_event_attrs[] = {571 &armv7_event_attr_sw_incr.attr.attr,572 &armv7_event_attr_l1i_cache_refill.attr.attr,573 &armv7_event_attr_l1i_tlb_refill.attr.attr,574 &armv7_event_attr_l1d_cache_refill.attr.attr,575 &armv7_event_attr_l1d_cache.attr.attr,576 &armv7_event_attr_l1d_tlb_refill.attr.attr,577 &armv7_event_attr_ld_retired.attr.attr,578 &armv7_event_attr_st_retired.attr.attr,579 &armv7_event_attr_inst_retired.attr.attr,580 &armv7_event_attr_exc_taken.attr.attr,581 &armv7_event_attr_exc_return.attr.attr,582 &armv7_event_attr_cid_write_retired.attr.attr,583 &armv7_event_attr_pc_write_retired.attr.attr,584 &armv7_event_attr_br_immed_retired.attr.attr,585 &armv7_event_attr_br_return_retired.attr.attr,586 &armv7_event_attr_unaligned_ldst_retired.attr.attr,587 &armv7_event_attr_br_mis_pred.attr.attr,588 &armv7_event_attr_cpu_cycles.attr.attr,589 &armv7_event_attr_br_pred.attr.attr,590 NULL,591};592 593static struct attribute_group armv7_pmuv1_events_attr_group = {594 .name = "events",595 .attrs = armv7_pmuv1_event_attrs,596};597 598ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS);599ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS);600ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB);601ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS);602ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL);603ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB);604ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS);605ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR);606ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC);607ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE);608ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES);609 610static struct attribute *armv7_pmuv2_event_attrs[] = {611 &armv7_event_attr_sw_incr.attr.attr,612 &armv7_event_attr_l1i_cache_refill.attr.attr,613 &armv7_event_attr_l1i_tlb_refill.attr.attr,614 &armv7_event_attr_l1d_cache_refill.attr.attr,615 &armv7_event_attr_l1d_cache.attr.attr,616 &armv7_event_attr_l1d_tlb_refill.attr.attr,617 &armv7_event_attr_ld_retired.attr.attr,618 &armv7_event_attr_st_retired.attr.attr,619 &armv7_event_attr_inst_retired.attr.attr,620 &armv7_event_attr_exc_taken.attr.attr,621 &armv7_event_attr_exc_return.attr.attr,622 &armv7_event_attr_cid_write_retired.attr.attr,623 &armv7_event_attr_pc_write_retired.attr.attr,624 &armv7_event_attr_br_immed_retired.attr.attr,625 &armv7_event_attr_br_return_retired.attr.attr,626 &armv7_event_attr_unaligned_ldst_retired.attr.attr,627 &armv7_event_attr_br_mis_pred.attr.attr,628 &armv7_event_attr_cpu_cycles.attr.attr,629 &armv7_event_attr_br_pred.attr.attr,630 &armv7_event_attr_mem_access.attr.attr,631 &armv7_event_attr_l1i_cache.attr.attr,632 &armv7_event_attr_l1d_cache_wb.attr.attr,633 &armv7_event_attr_l2d_cache.attr.attr,634 &armv7_event_attr_l2d_cache_refill.attr.attr,635 &armv7_event_attr_l2d_cache_wb.attr.attr,636 &armv7_event_attr_bus_access.attr.attr,637 &armv7_event_attr_memory_error.attr.attr,638 &armv7_event_attr_inst_spec.attr.attr,639 &armv7_event_attr_ttbr_write_retired.attr.attr,640 &armv7_event_attr_bus_cycles.attr.attr,641 NULL,642};643 644static struct attribute_group armv7_pmuv2_events_attr_group = {645 .name = "events",646 .attrs = armv7_pmuv2_event_attrs,647};648 649/*650 * Perf Events' indices651 */652#define ARMV7_IDX_CYCLE_COUNTER 31653#define ARMV7_IDX_COUNTER_MAX 31654/*655 * ARMv7 low level PMNC access656 */657 658/*659 * Per-CPU PMNC: config reg660 */661#define ARMV7_PMNC_E (1 << 0) /* Enable all counters */662#define ARMV7_PMNC_P (1 << 1) /* Reset all counters */663#define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */664#define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */665#define ARMV7_PMNC_X (1 << 4) /* Export to ETM */666#define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/667#define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */668#define ARMV7_PMNC_N_MASK 0x1f669#define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */670 671/*672 * FLAG: counters overflow flag status reg673 */674#define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */675#define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK676 677/*678 * PMXEVTYPER: Event selection reg679 */680#define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */681#define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */682 683/*684 * Event filters for PMUv2685 */686#define ARMV7_EXCLUDE_PL1 BIT(31)687#define ARMV7_EXCLUDE_USER BIT(30)688#define ARMV7_INCLUDE_HYP BIT(27)689 690/*691 * Secure debug enable reg692 */693#define ARMV7_SDER_SUNIDEN BIT(1) /* Permit non-invasive debug */694 695static inline u32 armv7_pmnc_read(void)696{697 u32 val;698 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));699 return val;700}701 702static inline void armv7_pmnc_write(u32 val)703{704 val &= ARMV7_PMNC_MASK;705 isb();706 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));707}708 709static inline int armv7_pmnc_has_overflowed(u32 pmnc)710{711 return pmnc & ARMV7_OVERFLOWED_MASK;712}713 714static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)715{716 return test_bit(idx, cpu_pmu->cntr_mask);717}718 719static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)720{721 return pmnc & BIT(idx);722}723 724static inline void armv7_pmnc_select_counter(int idx)725{726 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (idx));727 isb();728}729 730static inline u64 armv7pmu_read_counter(struct perf_event *event)731{732 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);733 struct hw_perf_event *hwc = &event->hw;734 int idx = hwc->idx;735 u32 value = 0;736 737 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {738 pr_err("CPU%u reading wrong counter %d\n",739 smp_processor_id(), idx);740 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {741 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));742 } else {743 armv7_pmnc_select_counter(idx);744 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));745 }746 747 return value;748}749 750static inline void armv7pmu_write_counter(struct perf_event *event, u64 value)751{752 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);753 struct hw_perf_event *hwc = &event->hw;754 int idx = hwc->idx;755 756 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {757 pr_err("CPU%u writing wrong counter %d\n",758 smp_processor_id(), idx);759 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {760 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value));761 } else {762 armv7_pmnc_select_counter(idx);763 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value));764 }765}766 767static inline void armv7_pmnc_write_evtsel(int idx, u32 val)768{769 armv7_pmnc_select_counter(idx);770 val &= ARMV7_EVTYPE_MASK;771 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));772}773 774static inline void armv7_pmnc_enable_counter(int idx)775{776 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(idx)));777}778 779static inline void armv7_pmnc_disable_counter(int idx)780{781 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(idx)));782}783 784static inline void armv7_pmnc_enable_intens(int idx)785{786 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(idx)));787}788 789static inline void armv7_pmnc_disable_intens(int idx)790{791 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(idx)));792 isb();793 /* Clear the overflow flag in case an interrupt is pending. */794 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(idx)));795 isb();796}797 798static inline u32 armv7_pmnc_getreset_flags(void)799{800 u32 val;801 802 /* Read */803 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));804 805 /* Write to clear flags */806 val &= ARMV7_FLAG_MASK;807 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));808 809 return val;810}811 812#ifdef DEBUG813static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)814{815 u32 val;816 unsigned int cnt;817 818 pr_info("PMNC registers dump:\n");819 820 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));821 pr_info("PMNC =0x%08x\n", val);822 823 asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));824 pr_info("CNTENS=0x%08x\n", val);825 826 asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));827 pr_info("INTENS=0x%08x\n", val);828 829 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));830 pr_info("FLAGS =0x%08x\n", val);831 832 asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));833 pr_info("SELECT=0x%08x\n", val);834 835 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));836 pr_info("CCNT =0x%08x\n", val);837 838 for_each_set_bit(cnt, cpu_pmu->cntr_mask, ARMV7_IDX_COUNTER_MAX) {839 armv7_pmnc_select_counter(cnt);840 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));841 pr_info("CNT[%d] count =0x%08x\n", cnt, val);842 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));843 pr_info("CNT[%d] evtsel=0x%08x\n", cnt, val);844 }845}846#endif847 848static void armv7pmu_enable_event(struct perf_event *event)849{850 struct hw_perf_event *hwc = &event->hw;851 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);852 int idx = hwc->idx;853 854 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {855 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",856 smp_processor_id(), idx);857 return;858 }859 860 /*861 * Enable counter and interrupt, and set the counter to count862 * the event that we're interested in.863 */864 865 /*866 * Disable counter867 */868 armv7_pmnc_disable_counter(idx);869 870 /*871 * Set event (if destined for PMNx counters)872 * We only need to set the event for the cycle counter if we873 * have the ability to perform event filtering.874 */875 if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)876 armv7_pmnc_write_evtsel(idx, hwc->config_base);877 878 /*879 * Enable interrupt for this counter880 */881 armv7_pmnc_enable_intens(idx);882 883 /*884 * Enable counter885 */886 armv7_pmnc_enable_counter(idx);887}888 889static void armv7pmu_disable_event(struct perf_event *event)890{891 struct hw_perf_event *hwc = &event->hw;892 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);893 int idx = hwc->idx;894 895 if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {896 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",897 smp_processor_id(), idx);898 return;899 }900 901 /*902 * Disable counter and interrupt903 */904 905 /*906 * Disable counter907 */908 armv7_pmnc_disable_counter(idx);909 910 /*911 * Disable interrupt for this counter912 */913 armv7_pmnc_disable_intens(idx);914}915 916static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu)917{918 u32 pmnc;919 struct perf_sample_data data;920 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);921 struct pt_regs *regs;922 int idx;923 924 /*925 * Get and reset the IRQ flags926 */927 pmnc = armv7_pmnc_getreset_flags();928 929 /*930 * Did an overflow occur?931 */932 if (!armv7_pmnc_has_overflowed(pmnc))933 return IRQ_NONE;934 935 /*936 * Handle the counter(s) overflow(s)937 */938 regs = get_irq_regs();939 940 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMPMU_MAX_HWEVENTS) {941 struct perf_event *event = cpuc->events[idx];942 struct hw_perf_event *hwc;943 944 /* Ignore if we don't have an event. */945 if (!event)946 continue;947 948 /*949 * We have a single interrupt for all counters. Check that950 * each counter has overflowed before we process it.951 */952 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))953 continue;954 955 hwc = &event->hw;956 armpmu_event_update(event);957 perf_sample_data_init(&data, 0, hwc->last_period);958 if (!armpmu_event_set_period(event))959 continue;960 961 if (perf_event_overflow(event, &data, regs))962 cpu_pmu->disable(event);963 }964 965 /*966 * Handle the pending perf events.967 *968 * Note: this call *must* be run with interrupts disabled. For969 * platforms that can have the PMU interrupts raised as an NMI, this970 * will not work.971 */972 irq_work_run();973 974 return IRQ_HANDLED;975}976 977static void armv7pmu_start(struct arm_pmu *cpu_pmu)978{979 /* Enable all counters */980 armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);981}982 983static void armv7pmu_stop(struct arm_pmu *cpu_pmu)984{985 /* Disable all counters */986 armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);987}988 989static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,990 struct perf_event *event)991{992 int idx;993 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);994 struct hw_perf_event *hwc = &event->hw;995 unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;996 997 /* Always place a cycle counter into the cycle counter. */998 if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {999 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))1000 return -EAGAIN;1001 1002 return ARMV7_IDX_CYCLE_COUNTER;1003 }1004 1005 /*1006 * For anything other than a cycle counter, try and use1007 * the events counters1008 */1009 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMV7_IDX_COUNTER_MAX) {1010 if (!test_and_set_bit(idx, cpuc->used_mask))1011 return idx;1012 }1013 1014 /* The counters are all in use. */1015 return -EAGAIN;1016}1017 1018static void armv7pmu_clear_event_idx(struct pmu_hw_events *cpuc,1019 struct perf_event *event)1020{1021 clear_bit(event->hw.idx, cpuc->used_mask);1022}1023 1024/*1025 * Add an event filter to a given event. This will only work for PMUv2 PMUs.1026 */1027static int armv7pmu_set_event_filter(struct hw_perf_event *event,1028 struct perf_event_attr *attr)1029{1030 unsigned long config_base = 0;1031 1032 if (attr->exclude_idle) {1033 pr_debug("ARM performance counters do not support mode exclusion\n");1034 return -EOPNOTSUPP;1035 }1036 if (attr->exclude_user)1037 config_base |= ARMV7_EXCLUDE_USER;1038 if (attr->exclude_kernel)1039 config_base |= ARMV7_EXCLUDE_PL1;1040 if (!attr->exclude_hv)1041 config_base |= ARMV7_INCLUDE_HYP;1042 1043 /*1044 * Install the filter into config_base as this is used to1045 * construct the event type.1046 */1047 event->config_base = config_base;1048 1049 return 0;1050}1051 1052static void armv7pmu_reset(void *info)1053{1054 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;1055 u32 idx, val;1056 1057 if (cpu_pmu->secure_access) {1058 asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val));1059 val |= ARMV7_SDER_SUNIDEN;1060 asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val));1061 }1062 1063 /* The counter and interrupt enable registers are unknown at reset. */1064 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMPMU_MAX_HWEVENTS) {1065 armv7_pmnc_disable_counter(idx);1066 armv7_pmnc_disable_intens(idx);1067 }1068 1069 /* Initialize & Reset PMNC: C and P bits */1070 armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);1071}1072 1073static int armv7_a8_map_event(struct perf_event *event)1074{1075 return armpmu_map_event(event, &armv7_a8_perf_map,1076 &armv7_a8_perf_cache_map, 0xFF);1077}1078 1079static int armv7_a9_map_event(struct perf_event *event)1080{1081 return armpmu_map_event(event, &armv7_a9_perf_map,1082 &armv7_a9_perf_cache_map, 0xFF);1083}1084 1085static int armv7_a5_map_event(struct perf_event *event)1086{1087 return armpmu_map_event(event, &armv7_a5_perf_map,1088 &armv7_a5_perf_cache_map, 0xFF);1089}1090 1091static int armv7_a15_map_event(struct perf_event *event)1092{1093 return armpmu_map_event(event, &armv7_a15_perf_map,1094 &armv7_a15_perf_cache_map, 0xFF);1095}1096 1097static int armv7_a7_map_event(struct perf_event *event)1098{1099 return armpmu_map_event(event, &armv7_a7_perf_map,1100 &armv7_a7_perf_cache_map, 0xFF);1101}1102 1103static int armv7_a12_map_event(struct perf_event *event)1104{1105 return armpmu_map_event(event, &armv7_a12_perf_map,1106 &armv7_a12_perf_cache_map, 0xFF);1107}1108 1109static int krait_map_event(struct perf_event *event)1110{1111 return armpmu_map_event(event, &krait_perf_map,1112 &krait_perf_cache_map, 0xFFFFF);1113}1114 1115static int krait_map_event_no_branch(struct perf_event *event)1116{1117 return armpmu_map_event(event, &krait_perf_map_no_branch,1118 &krait_perf_cache_map, 0xFFFFF);1119}1120 1121static int scorpion_map_event(struct perf_event *event)1122{1123 return armpmu_map_event(event, &scorpion_perf_map,1124 &scorpion_perf_cache_map, 0xFFFFF);1125}1126 1127static void armv7pmu_init(struct arm_pmu *cpu_pmu)1128{1129 cpu_pmu->handle_irq = armv7pmu_handle_irq;1130 cpu_pmu->enable = armv7pmu_enable_event;1131 cpu_pmu->disable = armv7pmu_disable_event;1132 cpu_pmu->read_counter = armv7pmu_read_counter;1133 cpu_pmu->write_counter = armv7pmu_write_counter;1134 cpu_pmu->get_event_idx = armv7pmu_get_event_idx;1135 cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx;1136 cpu_pmu->start = armv7pmu_start;1137 cpu_pmu->stop = armv7pmu_stop;1138 cpu_pmu->reset = armv7pmu_reset;1139};1140 1141static void armv7_read_num_pmnc_events(void *info)1142{1143 int nb_cnt;1144 struct arm_pmu *cpu_pmu = info;1145 1146 /* Read the nb of CNTx counters supported from PMNC */1147 nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;1148 bitmap_set(cpu_pmu->cntr_mask, 0, nb_cnt);1149 1150 /* Add the CPU cycles counter */1151 set_bit(ARMV7_IDX_CYCLE_COUNTER, cpu_pmu->cntr_mask);1152}1153 1154static int armv7_probe_num_events(struct arm_pmu *arm_pmu)1155{1156 return smp_call_function_any(&arm_pmu->supported_cpus,1157 armv7_read_num_pmnc_events,1158 arm_pmu, 1);1159}1160 1161static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)1162{1163 armv7pmu_init(cpu_pmu);1164 cpu_pmu->name = "armv7_cortex_a8";1165 cpu_pmu->map_event = armv7_a8_map_event;1166 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =1167 &armv7_pmuv1_events_attr_group;1168 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =1169 &armv7_pmu_format_attr_group;1170 return armv7_probe_num_events(cpu_pmu);1171}1172 1173static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)1174{1175 armv7pmu_init(cpu_pmu);1176 cpu_pmu->name = "armv7_cortex_a9";1177 cpu_pmu->map_event = armv7_a9_map_event;1178 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =1179 &armv7_pmuv1_events_attr_group;1180 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =1181 &armv7_pmu_format_attr_group;1182 return armv7_probe_num_events(cpu_pmu);1183}1184 1185static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)1186{1187 armv7pmu_init(cpu_pmu);1188 cpu_pmu->name = "armv7_cortex_a5";1189 cpu_pmu->map_event = armv7_a5_map_event;1190 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =1191 &armv7_pmuv1_events_attr_group;1192 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =1193 &armv7_pmu_format_attr_group;1194 return armv7_probe_num_events(cpu_pmu);1195}1196 1197static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)1198{1199 armv7pmu_init(cpu_pmu);1200 cpu_pmu->name = "armv7_cortex_a15";1201 cpu_pmu->map_event = armv7_a15_map_event;1202 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;1203 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =1204 &armv7_pmuv2_events_attr_group;1205 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =1206 &armv7_pmu_format_attr_group;1207 return armv7_probe_num_events(cpu_pmu);1208}1209 1210static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)1211{1212 armv7pmu_init(cpu_pmu);1213 cpu_pmu->name = "armv7_cortex_a7";1214 cpu_pmu->map_event = armv7_a7_map_event;1215 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;1216 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =1217 &armv7_pmuv2_events_attr_group;1218 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =1219 &armv7_pmu_format_attr_group;1220 return armv7_probe_num_events(cpu_pmu);1221}1222 1223static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)1224{1225 armv7pmu_init(cpu_pmu);1226 cpu_pmu->name = "armv7_cortex_a12";1227 cpu_pmu->map_event = armv7_a12_map_event;1228 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;1229 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =1230 &armv7_pmuv2_events_attr_group;1231 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =1232 &armv7_pmu_format_attr_group;1233 return armv7_probe_num_events(cpu_pmu);1234}1235 1236static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)1237{1238 int ret = armv7_a12_pmu_init(cpu_pmu);1239 cpu_pmu->name = "armv7_cortex_a17";1240 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =1241 &armv7_pmuv2_events_attr_group;1242 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =1243 &armv7_pmu_format_attr_group;1244 return ret;1245}1246 1247/*1248 * Krait Performance Monitor Region Event Selection Register (PMRESRn)1249 *1250 * 31 30 24 16 8 01251 * +--------------------------------+1252 * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 01253 * +--------------------------------+1254 * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 11255 * +--------------------------------+1256 * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 21257 * +--------------------------------+1258 * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ?1259 * +--------------------------------+1260 * EN | G=3 | G=2 | G=1 | G=01261 *1262 * Event Encoding:1263 *1264 * hwc->config_base = 0xNRCCG1265 *1266 * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)1267 * R = region register1268 * CC = class of events the group G is choosing from1269 * G = group or particular event1270 *1271 * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 21272 *1273 * A region (R) corresponds to a piece of the CPU (execution unit, instruction1274 * unit, etc.) while the event code (CC) corresponds to a particular class of1275 * events (interrupts for example). An event code is broken down into1276 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for1277 * example).1278 */1279 1280#define KRAIT_EVENT (1 << 16)1281#define VENUM_EVENT (2 << 16)1282#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT)1283#define PMRESRn_EN BIT(31)1284 1285#define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */1286#define EVENT_GROUP(event) ((event) & 0xf) /* G */1287#define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */1288#define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */1289#define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */1290 1291static u32 krait_read_pmresrn(int n)1292{1293 u32 val;1294 1295 switch (n) {1296 case 0:1297 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));1298 break;1299 case 1:1300 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));1301 break;1302 case 2:1303 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));1304 break;1305 default:1306 BUG(); /* Should be validated in krait_pmu_get_event_idx() */1307 }1308 1309 return val;1310}1311 1312static void krait_write_pmresrn(int n, u32 val)1313{1314 switch (n) {1315 case 0:1316 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));1317 break;1318 case 1:1319 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));1320 break;1321 case 2:1322 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));1323 break;1324 default:1325 BUG(); /* Should be validated in krait_pmu_get_event_idx() */1326 }1327}1328 1329static u32 venum_read_pmresr(void)1330{1331 u32 val;1332 asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));1333 return val;1334}1335 1336static void venum_write_pmresr(u32 val)1337{1338 asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));1339}1340 1341static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)1342{1343 u32 venum_new_val;1344 u32 fp_new_val;1345 1346 BUG_ON(preemptible());1347 /* CPACR Enable CP10 and CP11 access */1348 *venum_orig_val = get_copro_access();1349 venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);1350 set_copro_access(venum_new_val);1351 1352 /* Enable FPEXC */1353 *fp_orig_val = fmrx(FPEXC);1354 fp_new_val = *fp_orig_val | FPEXC_EN;1355 fmxr(FPEXC, fp_new_val);1356}1357 1358static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)1359{1360 BUG_ON(preemptible());1361 /* Restore FPEXC */1362 fmxr(FPEXC, fp_orig_val);1363 isb();1364 /* Restore CPACR */1365 set_copro_access(venum_orig_val);1366}1367 1368static u32 krait_get_pmresrn_event(unsigned int region)1369{1370 static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,1371 KRAIT_PMRESR1_GROUP0,1372 KRAIT_PMRESR2_GROUP0 };1373 return pmresrn_table[region];1374}1375 1376static void krait_evt_setup(int idx, u32 config_base)1377{1378 u32 val;1379 u32 mask;1380 u32 vval, fval;1381 unsigned int region = EVENT_REGION(config_base);1382 unsigned int group = EVENT_GROUP(config_base);1383 unsigned int code = EVENT_CODE(config_base);1384 unsigned int group_shift;1385 bool venum_event = EVENT_VENUM(config_base);1386 1387 group_shift = group * 8;1388 mask = 0xff << group_shift;1389 1390 /* Configure evtsel for the region and group */1391 if (venum_event)1392 val = KRAIT_VPMRESR0_GROUP0;1393 else1394 val = krait_get_pmresrn_event(region);1395 val += group;1396 /* Mix in mode-exclusion bits */1397 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);1398 armv7_pmnc_write_evtsel(idx, val);1399 1400 if (venum_event) {1401 venum_pre_pmresr(&vval, &fval);1402 val = venum_read_pmresr();1403 val &= ~mask;1404 val |= code << group_shift;1405 val |= PMRESRn_EN;1406 venum_write_pmresr(val);1407 venum_post_pmresr(vval, fval);1408 } else {1409 val = krait_read_pmresrn(region);1410 val &= ~mask;1411 val |= code << group_shift;1412 val |= PMRESRn_EN;1413 krait_write_pmresrn(region, val);1414 }1415}1416 1417static u32 clear_pmresrn_group(u32 val, int group)1418{1419 u32 mask;1420 int group_shift;1421 1422 group_shift = group * 8;1423 mask = 0xff << group_shift;1424 val &= ~mask;1425 1426 /* Don't clear enable bit if entire region isn't disabled */1427 if (val & ~PMRESRn_EN)1428 return val |= PMRESRn_EN;1429 1430 return 0;1431}1432 1433static void krait_clearpmu(u32 config_base)1434{1435 u32 val;1436 u32 vval, fval;1437 unsigned int region = EVENT_REGION(config_base);1438 unsigned int group = EVENT_GROUP(config_base);1439 bool venum_event = EVENT_VENUM(config_base);1440 1441 if (venum_event) {1442 venum_pre_pmresr(&vval, &fval);1443 val = venum_read_pmresr();1444 val = clear_pmresrn_group(val, group);1445 venum_write_pmresr(val);1446 venum_post_pmresr(vval, fval);1447 } else {1448 val = krait_read_pmresrn(region);1449 val = clear_pmresrn_group(val, group);1450 krait_write_pmresrn(region, val);1451 }1452}1453 1454static void krait_pmu_disable_event(struct perf_event *event)1455{1456 struct hw_perf_event *hwc = &event->hw;1457 int idx = hwc->idx;1458 1459 /* Disable counter and interrupt */1460 1461 /* Disable counter */1462 armv7_pmnc_disable_counter(idx);1463 1464 /*1465 * Clear pmresr code (if destined for PMNx counters)1466 */1467 if (hwc->config_base & KRAIT_EVENT_MASK)1468 krait_clearpmu(hwc->config_base);1469 1470 /* Disable interrupt for this counter */1471 armv7_pmnc_disable_intens(idx);1472}1473 1474static void krait_pmu_enable_event(struct perf_event *event)1475{1476 struct hw_perf_event *hwc = &event->hw;1477 int idx = hwc->idx;1478 1479 /*1480 * Enable counter and interrupt, and set the counter to count1481 * the event that we're interested in.1482 */1483 1484 /* Disable counter */1485 armv7_pmnc_disable_counter(idx);1486 1487 /*1488 * Set event (if destined for PMNx counters)1489 * We set the event for the cycle counter because we1490 * have the ability to perform event filtering.1491 */1492 if (hwc->config_base & KRAIT_EVENT_MASK)1493 krait_evt_setup(idx, hwc->config_base);1494 else1495 armv7_pmnc_write_evtsel(idx, hwc->config_base);1496 1497 /* Enable interrupt for this counter */1498 armv7_pmnc_enable_intens(idx);1499 1500 /* Enable counter */1501 armv7_pmnc_enable_counter(idx);1502}1503 1504static void krait_pmu_reset(void *info)1505{1506 u32 vval, fval;1507 struct arm_pmu *cpu_pmu = info;1508 u32 idx;1509 1510 armv7pmu_reset(info);1511 1512 /* Clear all pmresrs */1513 krait_write_pmresrn(0, 0);1514 krait_write_pmresrn(1, 0);1515 krait_write_pmresrn(2, 0);1516 1517 venum_pre_pmresr(&vval, &fval);1518 venum_write_pmresr(0);1519 venum_post_pmresr(vval, fval);1520 1521 /* Reset PMxEVNCTCR to sane default */1522 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMV7_IDX_COUNTER_MAX) {1523 armv7_pmnc_select_counter(idx);1524 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));1525 }1526 1527}1528 1529static int krait_event_to_bit(struct perf_event *event, unsigned int region,1530 unsigned int group)1531{1532 int bit;1533 struct hw_perf_event *hwc = &event->hw;1534 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);1535 1536 if (hwc->config_base & VENUM_EVENT)1537 bit = KRAIT_VPMRESR0_GROUP0;1538 else1539 bit = krait_get_pmresrn_event(region);1540 bit -= krait_get_pmresrn_event(0);1541 bit += group;1542 /*1543 * Lower bits are reserved for use by the counters (see1544 * armv7pmu_get_event_idx() for more info)1545 */1546 bit += bitmap_weight(cpu_pmu->cntr_mask, ARMV7_IDX_COUNTER_MAX);1547 1548 return bit;1549}1550 1551/*1552 * We check for column exclusion constraints here.1553 * Two events cant use the same group within a pmresr register.1554 */1555static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,1556 struct perf_event *event)1557{1558 int idx;1559 int bit = -1;1560 struct hw_perf_event *hwc = &event->hw;1561 unsigned int region = EVENT_REGION(hwc->config_base);1562 unsigned int code = EVENT_CODE(hwc->config_base);1563 unsigned int group = EVENT_GROUP(hwc->config_base);1564 bool venum_event = EVENT_VENUM(hwc->config_base);1565 bool krait_event = EVENT_CPU(hwc->config_base);1566 1567 if (venum_event || krait_event) {1568 /* Ignore invalid events */1569 if (group > 3 || region > 2)1570 return -EINVAL;1571 if (venum_event && (code & 0xe0))1572 return -EINVAL;1573 1574 bit = krait_event_to_bit(event, region, group);1575 if (test_and_set_bit(bit, cpuc->used_mask))1576 return -EAGAIN;1577 }1578 1579 idx = armv7pmu_get_event_idx(cpuc, event);1580 if (idx < 0 && bit >= 0)1581 clear_bit(bit, cpuc->used_mask);1582 1583 return idx;1584}1585 1586static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,1587 struct perf_event *event)1588{1589 int bit;1590 struct hw_perf_event *hwc = &event->hw;1591 unsigned int region = EVENT_REGION(hwc->config_base);1592 unsigned int group = EVENT_GROUP(hwc->config_base);1593 bool venum_event = EVENT_VENUM(hwc->config_base);1594 bool krait_event = EVENT_CPU(hwc->config_base);1595 1596 armv7pmu_clear_event_idx(cpuc, event);1597 if (venum_event || krait_event) {1598 bit = krait_event_to_bit(event, region, group);1599 clear_bit(bit, cpuc->used_mask);1600 }1601}1602 1603static int krait_pmu_init(struct arm_pmu *cpu_pmu)1604{1605 armv7pmu_init(cpu_pmu);1606 cpu_pmu->name = "armv7_krait";1607 /* Some early versions of Krait don't support PC write events */1608 if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,1609 "qcom,no-pc-write"))1610 cpu_pmu->map_event = krait_map_event_no_branch;1611 else1612 cpu_pmu->map_event = krait_map_event;1613 cpu_pmu->set_event_filter = armv7pmu_set_event_filter;1614 cpu_pmu->reset = krait_pmu_reset;1615 cpu_pmu->enable = krait_pmu_enable_event;1616 cpu_pmu->disable = krait_pmu_disable_event;1617 cpu_pmu->get_event_idx = krait_pmu_get_event_idx;1618 cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;1619 return armv7_probe_num_events(cpu_pmu);1620}1621 1622/*1623 * Scorpion Local Performance Monitor Register (LPMn)1624 *1625 * 31 30 24 16 8 01626 * +--------------------------------+1627 * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 01628 * +--------------------------------+1629 * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 11630 * +--------------------------------+1631 * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 21632 * +--------------------------------+1633 * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 31634 * +--------------------------------+1635 * VLPM | EN | CC | CC | CC | CC | N = 2, R = ?1636 * +--------------------------------+1637 * EN | G=3 | G=2 | G=1 | G=01638 *1639 *1640 * Event Encoding:1641 *1642 * hwc->config_base = 0xNRCCG1643 *1644 * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)1645 * R = region register1646 * CC = class of events the group G is choosing from1647 * G = group or particular event1648 *1649 * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 21650 *1651 * A region (R) corresponds to a piece of the CPU (execution unit, instruction1652 * unit, etc.) while the event code (CC) corresponds to a particular class of1653 * events (interrupts for example). An event code is broken down into1654 * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for1655 * example).1656 */1657 1658static u32 scorpion_read_pmresrn(int n)1659{1660 u32 val;1661 1662 switch (n) {1663 case 0:1664 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));1665 break;1666 case 1:1667 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));1668 break;1669 case 2:1670 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));1671 break;1672 case 3:1673 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));1674 break;1675 default:1676 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */1677 }1678 1679 return val;1680}1681 1682static void scorpion_write_pmresrn(int n, u32 val)1683{1684 switch (n) {1685 case 0:1686 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));1687 break;1688 case 1:1689 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));1690 break;1691 case 2:1692 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));1693 break;1694 case 3:1695 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));1696 break;1697 default:1698 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */1699 }1700}1701 1702static u32 scorpion_get_pmresrn_event(unsigned int region)1703{1704 static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,1705 SCORPION_LPM1_GROUP0,1706 SCORPION_LPM2_GROUP0,1707 SCORPION_L2LPM_GROUP0 };1708 return pmresrn_table[region];1709}1710 1711static void scorpion_evt_setup(int idx, u32 config_base)1712{1713 u32 val;1714 u32 mask;1715 u32 vval, fval;1716 unsigned int region = EVENT_REGION(config_base);1717 unsigned int group = EVENT_GROUP(config_base);1718 unsigned int code = EVENT_CODE(config_base);1719 unsigned int group_shift;1720 bool venum_event = EVENT_VENUM(config_base);1721 1722 group_shift = group * 8;1723 mask = 0xff << group_shift;1724 1725 /* Configure evtsel for the region and group */1726 if (venum_event)1727 val = SCORPION_VLPM_GROUP0;1728 else1729 val = scorpion_get_pmresrn_event(region);1730 val += group;1731 /* Mix in mode-exclusion bits */1732 val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);1733 armv7_pmnc_write_evtsel(idx, val);1734 1735 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));1736 1737 if (venum_event) {1738 venum_pre_pmresr(&vval, &fval);1739 val = venum_read_pmresr();1740 val &= ~mask;1741 val |= code << group_shift;1742 val |= PMRESRn_EN;1743 venum_write_pmresr(val);1744 venum_post_pmresr(vval, fval);1745 } else {1746 val = scorpion_read_pmresrn(region);1747 val &= ~mask;1748 val |= code << group_shift;1749 val |= PMRESRn_EN;1750 scorpion_write_pmresrn(region, val);1751 }1752}1753 1754static void scorpion_clearpmu(u32 config_base)1755{1756 u32 val;1757 u32 vval, fval;1758 unsigned int region = EVENT_REGION(config_base);1759 unsigned int group = EVENT_GROUP(config_base);1760 bool venum_event = EVENT_VENUM(config_base);1761 1762 if (venum_event) {1763 venum_pre_pmresr(&vval, &fval);1764 val = venum_read_pmresr();1765 val = clear_pmresrn_group(val, group);1766 venum_write_pmresr(val);1767 venum_post_pmresr(vval, fval);1768 } else {1769 val = scorpion_read_pmresrn(region);1770 val = clear_pmresrn_group(val, group);1771 scorpion_write_pmresrn(region, val);1772 }1773}1774 1775static void scorpion_pmu_disable_event(struct perf_event *event)1776{1777 struct hw_perf_event *hwc = &event->hw;1778 int idx = hwc->idx;1779 1780 /* Disable counter and interrupt */1781 1782 /* Disable counter */1783 armv7_pmnc_disable_counter(idx);1784 1785 /*1786 * Clear pmresr code (if destined for PMNx counters)1787 */1788 if (hwc->config_base & KRAIT_EVENT_MASK)1789 scorpion_clearpmu(hwc->config_base);1790 1791 /* Disable interrupt for this counter */1792 armv7_pmnc_disable_intens(idx);1793}1794 1795static void scorpion_pmu_enable_event(struct perf_event *event)1796{1797 struct hw_perf_event *hwc = &event->hw;1798 int idx = hwc->idx;1799 1800 /*1801 * Enable counter and interrupt, and set the counter to count1802 * the event that we're interested in.1803 */1804 1805 /* Disable counter */1806 armv7_pmnc_disable_counter(idx);1807 1808 /*1809 * Set event (if destined for PMNx counters)1810 * We don't set the event for the cycle counter because we1811 * don't have the ability to perform event filtering.1812 */1813 if (hwc->config_base & KRAIT_EVENT_MASK)1814 scorpion_evt_setup(idx, hwc->config_base);1815 else if (idx != ARMV7_IDX_CYCLE_COUNTER)1816 armv7_pmnc_write_evtsel(idx, hwc->config_base);1817 1818 /* Enable interrupt for this counter */1819 armv7_pmnc_enable_intens(idx);1820 1821 /* Enable counter */1822 armv7_pmnc_enable_counter(idx);1823}1824 1825static void scorpion_pmu_reset(void *info)1826{1827 u32 vval, fval;1828 struct arm_pmu *cpu_pmu = info;1829 u32 idx;1830 1831 armv7pmu_reset(info);1832 1833 /* Clear all pmresrs */1834 scorpion_write_pmresrn(0, 0);1835 scorpion_write_pmresrn(1, 0);1836 scorpion_write_pmresrn(2, 0);1837 scorpion_write_pmresrn(3, 0);1838 1839 venum_pre_pmresr(&vval, &fval);1840 venum_write_pmresr(0);1841 venum_post_pmresr(vval, fval);1842 1843 /* Reset PMxEVNCTCR to sane default */1844 for_each_set_bit(idx, cpu_pmu->cntr_mask, ARMV7_IDX_COUNTER_MAX) {1845 armv7_pmnc_select_counter(idx);1846 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));1847 }1848}1849 1850static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,1851 unsigned int group)1852{1853 int bit;1854 struct hw_perf_event *hwc = &event->hw;1855 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);1856 1857 if (hwc->config_base & VENUM_EVENT)1858 bit = SCORPION_VLPM_GROUP0;1859 else1860 bit = scorpion_get_pmresrn_event(region);1861 bit -= scorpion_get_pmresrn_event(0);1862 bit += group;1863 /*1864 * Lower bits are reserved for use by the counters (see1865 * armv7pmu_get_event_idx() for more info)1866 */1867 bit += bitmap_weight(cpu_pmu->cntr_mask, ARMV7_IDX_COUNTER_MAX);1868 1869 return bit;1870}1871 1872/*1873 * We check for column exclusion constraints here.1874 * Two events cant use the same group within a pmresr register.1875 */1876static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,1877 struct perf_event *event)1878{1879 int idx;1880 int bit = -1;1881 struct hw_perf_event *hwc = &event->hw;1882 unsigned int region = EVENT_REGION(hwc->config_base);1883 unsigned int group = EVENT_GROUP(hwc->config_base);1884 bool venum_event = EVENT_VENUM(hwc->config_base);1885 bool scorpion_event = EVENT_CPU(hwc->config_base);1886 1887 if (venum_event || scorpion_event) {1888 /* Ignore invalid events */1889 if (group > 3 || region > 3)1890 return -EINVAL;1891 1892 bit = scorpion_event_to_bit(event, region, group);1893 if (test_and_set_bit(bit, cpuc->used_mask))1894 return -EAGAIN;1895 }1896 1897 idx = armv7pmu_get_event_idx(cpuc, event);1898 if (idx < 0 && bit >= 0)1899 clear_bit(bit, cpuc->used_mask);1900 1901 return idx;1902}1903 1904static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,1905 struct perf_event *event)1906{1907 int bit;1908 struct hw_perf_event *hwc = &event->hw;1909 unsigned int region = EVENT_REGION(hwc->config_base);1910 unsigned int group = EVENT_GROUP(hwc->config_base);1911 bool venum_event = EVENT_VENUM(hwc->config_base);1912 bool scorpion_event = EVENT_CPU(hwc->config_base);1913 1914 armv7pmu_clear_event_idx(cpuc, event);1915 if (venum_event || scorpion_event) {1916 bit = scorpion_event_to_bit(event, region, group);1917 clear_bit(bit, cpuc->used_mask);1918 }1919}1920 1921static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)1922{1923 armv7pmu_init(cpu_pmu);1924 cpu_pmu->name = "armv7_scorpion";1925 cpu_pmu->map_event = scorpion_map_event;1926 cpu_pmu->reset = scorpion_pmu_reset;1927 cpu_pmu->enable = scorpion_pmu_enable_event;1928 cpu_pmu->disable = scorpion_pmu_disable_event;1929 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;1930 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;1931 return armv7_probe_num_events(cpu_pmu);1932}1933 1934static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)1935{1936 armv7pmu_init(cpu_pmu);1937 cpu_pmu->name = "armv7_scorpion_mp";1938 cpu_pmu->map_event = scorpion_map_event;1939 cpu_pmu->reset = scorpion_pmu_reset;1940 cpu_pmu->enable = scorpion_pmu_enable_event;1941 cpu_pmu->disable = scorpion_pmu_disable_event;1942 cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx;1943 cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;1944 return armv7_probe_num_events(cpu_pmu);1945}1946 1947static const struct of_device_id armv7_pmu_of_device_ids[] = {1948 {.compatible = "arm,cortex-a17-pmu", .data = armv7_a17_pmu_init},1949 {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init},1950 {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init},1951 {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init},1952 {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init},1953 {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init},1954 {.compatible = "arm,cortex-a5-pmu", .data = armv7_a5_pmu_init},1955 {.compatible = "qcom,krait-pmu", .data = krait_pmu_init},1956 {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init},1957 {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init},1958 {},1959};1960 1961static int armv7_pmu_device_probe(struct platform_device *pdev)1962{1963 return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids, NULL);1964}1965 1966static struct platform_driver armv7_pmu_driver = {1967 .driver = {1968 .name = "armv7-pmu",1969 .of_match_table = armv7_pmu_of_device_ids,1970 .suppress_bind_attrs = true,1971 },1972 .probe = armv7_pmu_device_probe,1973};1974 1975builtin_platform_driver(armv7_pmu_driver);1976