1714 lines · c
1// SPDX-License-Identifier: GPL-2.02// CCI Cache Coherent Interconnect PMU driver3// Copyright (C) 2013-2018 Arm Ltd.4// Author: Punit Agrawal <punit.agrawal@arm.com>, Suzuki Poulose <suzuki.poulose@arm.com>5 6#include <linux/arm-cci.h>7#include <linux/io.h>8#include <linux/interrupt.h>9#include <linux/module.h>10#include <linux/of.h>11#include <linux/perf_event.h>12#include <linux/platform_device.h>13#include <linux/slab.h>14#include <linux/spinlock.h>15 16#define DRIVER_NAME "ARM-CCI PMU"17 18#define CCI_PMCR 0x010019#define CCI_PID2 0x0fe820 21#define CCI_PMCR_CEN 0x0000000122#define CCI_PMCR_NCNT_MASK 0x0000f80023#define CCI_PMCR_NCNT_SHIFT 1124 25#define CCI_PID2_REV_MASK 0xf026#define CCI_PID2_REV_SHIFT 427 28#define CCI_PMU_EVT_SEL 0x00029#define CCI_PMU_CNTR 0x00430#define CCI_PMU_CNTR_CTRL 0x00831#define CCI_PMU_OVRFLW 0x00c32 33#define CCI_PMU_OVRFLW_FLAG 134 35#define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size)36#define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model))37#define CCI_PMU_CNTR_MASK ((1ULL << 32) - 1)38#define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1)39 40#define CCI_PMU_MAX_HW_CNTRS(model) \41 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs)42 43/* Types of interfaces that can generate events */44enum {45 CCI_IF_SLAVE,46 CCI_IF_MASTER,47#ifdef CONFIG_ARM_CCI5xx_PMU48 CCI_IF_GLOBAL,49#endif50 CCI_IF_MAX,51};52 53#define NUM_HW_CNTRS_CII_4XX 454#define NUM_HW_CNTRS_CII_5XX 855#define NUM_HW_CNTRS_MAX NUM_HW_CNTRS_CII_5XX56 57#define FIXED_HW_CNTRS_CII_4XX 158#define FIXED_HW_CNTRS_CII_5XX 059#define FIXED_HW_CNTRS_MAX FIXED_HW_CNTRS_CII_4XX60 61#define HW_CNTRS_MAX (NUM_HW_CNTRS_MAX + FIXED_HW_CNTRS_MAX)62 63struct event_range {64 u32 min;65 u32 max;66};67 68struct cci_pmu_hw_events {69 struct perf_event **events;70 unsigned long *used_mask;71 raw_spinlock_t pmu_lock;72};73 74struct cci_pmu;75/*76 * struct cci_pmu_model:77 * @fixed_hw_cntrs - Number of fixed event counters78 * @num_hw_cntrs - Maximum number of programmable event counters79 * @cntr_size - Size of an event counter mapping80 */81struct cci_pmu_model {82 char *name;83 u32 fixed_hw_cntrs;84 u32 num_hw_cntrs;85 u32 cntr_size;86 struct attribute **format_attrs;87 struct attribute **event_attrs;88 struct event_range event_ranges[CCI_IF_MAX];89 int (*validate_hw_event)(struct cci_pmu *, unsigned long);90 int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long);91 void (*write_counters)(struct cci_pmu *, unsigned long *);92};93 94static struct cci_pmu_model cci_pmu_models[];95 96struct cci_pmu {97 void __iomem *base;98 void __iomem *ctrl_base;99 struct pmu pmu;100 int cpu;101 int nr_irqs;102 int *irqs;103 unsigned long active_irqs;104 const struct cci_pmu_model *model;105 struct cci_pmu_hw_events hw_events;106 struct platform_device *plat_device;107 int num_cntrs;108 atomic_t active_events;109 struct mutex reserve_mutex;110};111 112#define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu))113 114static struct cci_pmu *g_cci_pmu;115 116enum cci_models {117#ifdef CONFIG_ARM_CCI400_PMU118 CCI400_R0,119 CCI400_R1,120#endif121#ifdef CONFIG_ARM_CCI5xx_PMU122 CCI500_R0,123 CCI550_R0,124#endif125 CCI_MODEL_MAX126};127 128static void pmu_write_counters(struct cci_pmu *cci_pmu,129 unsigned long *mask);130static ssize_t __maybe_unused cci_pmu_event_show(struct device *dev,131 struct device_attribute *attr, char *buf);132 133#define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \134 &((struct dev_ext_attribute[]) { \135 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \136 })[0].attr.attr137 138#define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \139 CCI_EXT_ATTR_ENTRY(_name, device_show_string, _config)140#define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \141 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config)142 143/* CCI400 PMU Specific definitions */144 145#ifdef CONFIG_ARM_CCI400_PMU146 147/* Port ids */148#define CCI400_PORT_S0 0149#define CCI400_PORT_S1 1150#define CCI400_PORT_S2 2151#define CCI400_PORT_S3 3152#define CCI400_PORT_S4 4153#define CCI400_PORT_M0 5154#define CCI400_PORT_M1 6155#define CCI400_PORT_M2 7156 157#define CCI400_R1_PX 5158 159/*160 * Instead of an event id to monitor CCI cycles, a dedicated counter is161 * provided. Use 0xff to represent CCI cycles and hope that no future revisions162 * make use of this event in hardware.163 */164enum cci400_perf_events {165 CCI400_PMU_CYCLES = 0xff166};167 168#define CCI400_PMU_CYCLE_CNTR_IDX 0169#define CCI400_PMU_CNTR0_IDX 1170 171/*172 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8173 * ports and bits 4:0 are event codes. There are different event codes174 * associated with each port type.175 *176 * Additionally, the range of events associated with the port types changed177 * between Rev0 and Rev1.178 *179 * The constants below define the range of valid codes for each port type for180 * the different revisions and are used to validate the event to be monitored.181 */182 183#define CCI400_PMU_EVENT_MASK 0xffUL184#define CCI400_PMU_EVENT_SOURCE_SHIFT 5185#define CCI400_PMU_EVENT_SOURCE_MASK 0x7186#define CCI400_PMU_EVENT_CODE_SHIFT 0187#define CCI400_PMU_EVENT_CODE_MASK 0x1f188#define CCI400_PMU_EVENT_SOURCE(event) \189 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \190 CCI400_PMU_EVENT_SOURCE_MASK)191#define CCI400_PMU_EVENT_CODE(event) \192 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK)193 194#define CCI400_R0_SLAVE_PORT_MIN_EV 0x00195#define CCI400_R0_SLAVE_PORT_MAX_EV 0x13196#define CCI400_R0_MASTER_PORT_MIN_EV 0x14197#define CCI400_R0_MASTER_PORT_MAX_EV 0x1a198 199#define CCI400_R1_SLAVE_PORT_MIN_EV 0x00200#define CCI400_R1_SLAVE_PORT_MAX_EV 0x14201#define CCI400_R1_MASTER_PORT_MIN_EV 0x00202#define CCI400_R1_MASTER_PORT_MAX_EV 0x11203 204#define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \205 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \206 (unsigned long)_config)207 208static ssize_t cci400_pmu_cycle_event_show(struct device *dev,209 struct device_attribute *attr, char *buf);210 211static struct attribute *cci400_pmu_format_attrs[] = {212 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),213 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"),214 NULL215};216 217static struct attribute *cci400_r0_pmu_event_attrs[] = {218 /* Slave events */219 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),220 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),221 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),222 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),223 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),224 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),225 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),226 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),227 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),228 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),229 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),230 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),231 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),232 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),233 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),234 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),235 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),236 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),237 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),238 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),239 /* Master events */240 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14),241 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15),242 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16),243 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17),244 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18),245 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19),246 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A),247 /* Special event for cycles counter */248 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),249 NULL250};251 252static struct attribute *cci400_r1_pmu_event_attrs[] = {253 /* Slave events */254 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),255 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),256 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),257 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),258 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),259 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),260 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),261 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),262 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),263 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),264 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),265 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),266 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),267 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),268 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),269 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),270 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),271 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),272 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),273 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),274 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14),275 /* Master events */276 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0),277 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1),278 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2),279 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3),280 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4),281 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5),282 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6),283 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7),284 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8),285 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9),286 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA),287 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB),288 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC),289 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD),290 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE),291 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF),292 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10),293 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11),294 /* Special event for cycles counter */295 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),296 NULL297};298 299static ssize_t cci400_pmu_cycle_event_show(struct device *dev,300 struct device_attribute *attr, char *buf)301{302 struct dev_ext_attribute *eattr = container_of(attr,303 struct dev_ext_attribute, attr);304 return sysfs_emit(buf, "config=0x%lx\n", (unsigned long)eattr->var);305}306 307static int cci400_get_event_idx(struct cci_pmu *cci_pmu,308 struct cci_pmu_hw_events *hw,309 unsigned long cci_event)310{311 int idx;312 313 /* cycles event idx is fixed */314 if (cci_event == CCI400_PMU_CYCLES) {315 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask))316 return -EAGAIN;317 318 return CCI400_PMU_CYCLE_CNTR_IDX;319 }320 321 for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx)322 if (!test_and_set_bit(idx, hw->used_mask))323 return idx;324 325 /* No counters available */326 return -EAGAIN;327}328 329static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event)330{331 u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event);332 u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event);333 int if_type;334 335 if (hw_event & ~CCI400_PMU_EVENT_MASK)336 return -ENOENT;337 338 if (hw_event == CCI400_PMU_CYCLES)339 return hw_event;340 341 switch (ev_source) {342 case CCI400_PORT_S0:343 case CCI400_PORT_S1:344 case CCI400_PORT_S2:345 case CCI400_PORT_S3:346 case CCI400_PORT_S4:347 /* Slave Interface */348 if_type = CCI_IF_SLAVE;349 break;350 case CCI400_PORT_M0:351 case CCI400_PORT_M1:352 case CCI400_PORT_M2:353 /* Master Interface */354 if_type = CCI_IF_MASTER;355 break;356 default:357 return -ENOENT;358 }359 360 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&361 ev_code <= cci_pmu->model->event_ranges[if_type].max)362 return hw_event;363 364 return -ENOENT;365}366 367static int probe_cci400_revision(struct cci_pmu *cci_pmu)368{369 int rev;370 rev = readl_relaxed(cci_pmu->ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK;371 rev >>= CCI_PID2_REV_SHIFT;372 373 if (rev < CCI400_R1_PX)374 return CCI400_R0;375 else376 return CCI400_R1;377}378 379static const struct cci_pmu_model *probe_cci_model(struct cci_pmu *cci_pmu)380{381 if (platform_has_secure_cci_access())382 return &cci_pmu_models[probe_cci400_revision(cci_pmu)];383 return NULL;384}385#else /* !CONFIG_ARM_CCI400_PMU */386static inline struct cci_pmu_model *probe_cci_model(struct cci_pmu *cci_pmu)387{388 return NULL;389}390#endif /* CONFIG_ARM_CCI400_PMU */391 392#ifdef CONFIG_ARM_CCI5xx_PMU393 394/*395 * CCI5xx PMU event id is an 9-bit value made of two parts.396 * bits [8:5] - Source for the event397 * bits [4:0] - Event code (specific to type of interface)398 *399 *400 */401 402/* Port ids */403#define CCI5xx_PORT_S0 0x0404#define CCI5xx_PORT_S1 0x1405#define CCI5xx_PORT_S2 0x2406#define CCI5xx_PORT_S3 0x3407#define CCI5xx_PORT_S4 0x4408#define CCI5xx_PORT_S5 0x5409#define CCI5xx_PORT_S6 0x6410 411#define CCI5xx_PORT_M0 0x8412#define CCI5xx_PORT_M1 0x9413#define CCI5xx_PORT_M2 0xa414#define CCI5xx_PORT_M3 0xb415#define CCI5xx_PORT_M4 0xc416#define CCI5xx_PORT_M5 0xd417#define CCI5xx_PORT_M6 0xe418 419#define CCI5xx_PORT_GLOBAL 0xf420 421#define CCI5xx_PMU_EVENT_MASK 0x1ffUL422#define CCI5xx_PMU_EVENT_SOURCE_SHIFT 0x5423#define CCI5xx_PMU_EVENT_SOURCE_MASK 0xf424#define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0425#define CCI5xx_PMU_EVENT_CODE_MASK 0x1f426 427#define CCI5xx_PMU_EVENT_SOURCE(event) \428 ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK)429#define CCI5xx_PMU_EVENT_CODE(event) \430 ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK)431 432#define CCI5xx_SLAVE_PORT_MIN_EV 0x00433#define CCI5xx_SLAVE_PORT_MAX_EV 0x1f434#define CCI5xx_MASTER_PORT_MIN_EV 0x00435#define CCI5xx_MASTER_PORT_MAX_EV 0x06436#define CCI5xx_GLOBAL_PORT_MIN_EV 0x00437#define CCI5xx_GLOBAL_PORT_MAX_EV 0x0f438 439 440#define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \441 CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \442 (unsigned long) _config)443 444static ssize_t cci5xx_pmu_global_event_show(struct device *dev,445 struct device_attribute *attr, char *buf);446 447static struct attribute *cci5xx_pmu_format_attrs[] = {448 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),449 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"),450 NULL,451};452 453static struct attribute *cci5xx_pmu_event_attrs[] = {454 /* Slave events */455 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0),456 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1),457 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2),458 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3),459 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4),460 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5),461 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6),462 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),463 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8),464 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9),465 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA),466 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB),467 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC),468 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD),469 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE),470 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF),471 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10),472 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11),473 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12),474 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13),475 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14),476 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15),477 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16),478 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17),479 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18),480 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19),481 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A),482 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B),483 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C),484 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D),485 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E),486 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F),487 488 /* Master events */489 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0),490 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1),491 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2),492 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3),493 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4),494 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5),495 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6),496 497 /* Global events */498 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0),499 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1),500 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2),501 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3),502 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4),503 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5),504 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6),505 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7),506 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8),507 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9),508 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA),509 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB),510 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC),511 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD),512 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_stall_tt_full, 0xE),513 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF),514 NULL515};516 517static ssize_t cci5xx_pmu_global_event_show(struct device *dev,518 struct device_attribute *attr, char *buf)519{520 struct dev_ext_attribute *eattr = container_of(attr,521 struct dev_ext_attribute, attr);522 /* Global events have single fixed source code */523 return sysfs_emit(buf, "event=0x%lx,source=0x%x\n",524 (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL);525}526 527/*528 * CCI500 provides 8 independent event counters that can count529 * any of the events available.530 * CCI500 PMU event source ids531 * 0x0-0x6 - Slave interfaces532 * 0x8-0xD - Master interfaces533 * 0xf - Global Events534 * 0x7,0xe - Reserved535 */536static int cci500_validate_hw_event(struct cci_pmu *cci_pmu,537 unsigned long hw_event)538{539 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);540 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);541 int if_type;542 543 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)544 return -ENOENT;545 546 switch (ev_source) {547 case CCI5xx_PORT_S0:548 case CCI5xx_PORT_S1:549 case CCI5xx_PORT_S2:550 case CCI5xx_PORT_S3:551 case CCI5xx_PORT_S4:552 case CCI5xx_PORT_S5:553 case CCI5xx_PORT_S6:554 if_type = CCI_IF_SLAVE;555 break;556 case CCI5xx_PORT_M0:557 case CCI5xx_PORT_M1:558 case CCI5xx_PORT_M2:559 case CCI5xx_PORT_M3:560 case CCI5xx_PORT_M4:561 case CCI5xx_PORT_M5:562 if_type = CCI_IF_MASTER;563 break;564 case CCI5xx_PORT_GLOBAL:565 if_type = CCI_IF_GLOBAL;566 break;567 default:568 return -ENOENT;569 }570 571 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&572 ev_code <= cci_pmu->model->event_ranges[if_type].max)573 return hw_event;574 575 return -ENOENT;576}577 578/*579 * CCI550 provides 8 independent event counters that can count580 * any of the events available.581 * CCI550 PMU event source ids582 * 0x0-0x6 - Slave interfaces583 * 0x8-0xe - Master interfaces584 * 0xf - Global Events585 * 0x7 - Reserved586 */587static int cci550_validate_hw_event(struct cci_pmu *cci_pmu,588 unsigned long hw_event)589{590 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);591 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);592 int if_type;593 594 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)595 return -ENOENT;596 597 switch (ev_source) {598 case CCI5xx_PORT_S0:599 case CCI5xx_PORT_S1:600 case CCI5xx_PORT_S2:601 case CCI5xx_PORT_S3:602 case CCI5xx_PORT_S4:603 case CCI5xx_PORT_S5:604 case CCI5xx_PORT_S6:605 if_type = CCI_IF_SLAVE;606 break;607 case CCI5xx_PORT_M0:608 case CCI5xx_PORT_M1:609 case CCI5xx_PORT_M2:610 case CCI5xx_PORT_M3:611 case CCI5xx_PORT_M4:612 case CCI5xx_PORT_M5:613 case CCI5xx_PORT_M6:614 if_type = CCI_IF_MASTER;615 break;616 case CCI5xx_PORT_GLOBAL:617 if_type = CCI_IF_GLOBAL;618 break;619 default:620 return -ENOENT;621 }622 623 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&624 ev_code <= cci_pmu->model->event_ranges[if_type].max)625 return hw_event;626 627 return -ENOENT;628}629 630#endif /* CONFIG_ARM_CCI5xx_PMU */631 632/*633 * Program the CCI PMU counters which have PERF_HES_ARCH set634 * with the event period and mark them ready before we enable635 * PMU.636 */637static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu)638{639 int i;640 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;641 DECLARE_BITMAP(mask, HW_CNTRS_MAX);642 643 bitmap_zero(mask, HW_CNTRS_MAX);644 for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) {645 struct perf_event *event = cci_hw->events[i];646 647 if (WARN_ON(!event))648 continue;649 650 /* Leave the events which are not counting */651 if (event->hw.state & PERF_HES_STOPPED)652 continue;653 if (event->hw.state & PERF_HES_ARCH) {654 __set_bit(i, mask);655 event->hw.state &= ~PERF_HES_ARCH;656 }657 }658 659 pmu_write_counters(cci_pmu, mask);660}661 662/* Should be called with cci_pmu->hw_events->pmu_lock held */663static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu)664{665 u32 val;666 667 /* Enable all the PMU counters. */668 val = readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) | CCI_PMCR_CEN;669 writel(val, cci_pmu->ctrl_base + CCI_PMCR);670}671 672/* Should be called with cci_pmu->hw_events->pmu_lock held */673static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu)674{675 cci_pmu_sync_counters(cci_pmu);676 __cci_pmu_enable_nosync(cci_pmu);677}678 679/* Should be called with cci_pmu->hw_events->pmu_lock held */680static void __cci_pmu_disable(struct cci_pmu *cci_pmu)681{682 u32 val;683 684 /* Disable all the PMU counters. */685 val = readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN;686 writel(val, cci_pmu->ctrl_base + CCI_PMCR);687}688 689static ssize_t cci_pmu_event_show(struct device *dev,690 struct device_attribute *attr, char *buf)691{692 struct dev_ext_attribute *eattr = container_of(attr,693 struct dev_ext_attribute, attr);694 /* source parameter is mandatory for normal PMU events */695 return sysfs_emit(buf, "source=?,event=0x%lx\n",696 (unsigned long)eattr->var);697}698 699static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx)700{701 return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu);702}703 704static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset)705{706 return readl_relaxed(cci_pmu->base +707 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);708}709 710static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value,711 int idx, unsigned int offset)712{713 writel_relaxed(value, cci_pmu->base +714 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);715}716 717static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx)718{719 pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL);720}721 722static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx)723{724 pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL);725}726 727static bool __maybe_unused728pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx)729{730 return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0;731}732 733static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event)734{735 pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL);736}737 738/*739 * For all counters on the CCI-PMU, disable any 'enabled' counters,740 * saving the changed counters in the mask, so that we can restore741 * it later using pmu_restore_counters. The mask is private to the742 * caller. We cannot rely on the used_mask maintained by the CCI_PMU743 * as it only tells us if the counter is assigned to perf_event or not.744 * The state of the perf_event cannot be locked by the PMU layer, hence745 * we check the individual counter status (which can be locked by746 * cci_pm->hw_events->pmu_lock).747 *748 * @mask should be initialised to empty by the caller.749 */750static void __maybe_unused751pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask)752{753 int i;754 755 for (i = 0; i < cci_pmu->num_cntrs; i++) {756 if (pmu_counter_is_enabled(cci_pmu, i)) {757 set_bit(i, mask);758 pmu_disable_counter(cci_pmu, i);759 }760 }761}762 763/*764 * Restore the status of the counters. Reversal of the pmu_save_counters().765 * For each counter set in the mask, enable the counter back.766 */767static void __maybe_unused768pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask)769{770 int i;771 772 for_each_set_bit(i, mask, cci_pmu->num_cntrs)773 pmu_enable_counter(cci_pmu, i);774}775 776/*777 * Returns the number of programmable counters actually implemented778 * by the cci779 */780static u32 pmu_get_max_counters(struct cci_pmu *cci_pmu)781{782 return (readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) &783 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT;784}785 786static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event)787{788 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);789 unsigned long cci_event = event->hw.config_base;790 int idx;791 792 if (cci_pmu->model->get_event_idx)793 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event);794 795 /* Generic code to find an unused idx from the mask */796 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++)797 if (!test_and_set_bit(idx, hw->used_mask))798 return idx;799 800 /* No counters available */801 return -EAGAIN;802}803 804static int pmu_map_event(struct perf_event *event)805{806 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);807 808 if (event->attr.type < PERF_TYPE_MAX ||809 !cci_pmu->model->validate_hw_event)810 return -ENOENT;811 812 return cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config);813}814 815static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler)816{817 int i;818 struct platform_device *pmu_device = cci_pmu->plat_device;819 820 if (unlikely(!pmu_device))821 return -ENODEV;822 823 if (cci_pmu->nr_irqs < 1) {824 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n");825 return -ENODEV;826 }827 828 /*829 * Register all available CCI PMU interrupts. In the interrupt handler830 * we iterate over the counters checking for interrupt source (the831 * overflowing counter) and clear it.832 *833 * This should allow handling of non-unique interrupt for the counters.834 */835 for (i = 0; i < cci_pmu->nr_irqs; i++) {836 int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED,837 "arm-cci-pmu", cci_pmu);838 if (err) {839 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",840 cci_pmu->irqs[i]);841 return err;842 }843 844 set_bit(i, &cci_pmu->active_irqs);845 }846 847 return 0;848}849 850static void pmu_free_irq(struct cci_pmu *cci_pmu)851{852 int i;853 854 for (i = 0; i < cci_pmu->nr_irqs; i++) {855 if (!test_and_clear_bit(i, &cci_pmu->active_irqs))856 continue;857 858 free_irq(cci_pmu->irqs[i], cci_pmu);859 }860}861 862static u32 pmu_read_counter(struct perf_event *event)863{864 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);865 struct hw_perf_event *hw_counter = &event->hw;866 int idx = hw_counter->idx;867 u32 value;868 869 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {870 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);871 return 0;872 }873 value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR);874 875 return value;876}877 878static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx)879{880 pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR);881}882 883static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)884{885 int i;886 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;887 888 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {889 struct perf_event *event = cci_hw->events[i];890 891 if (WARN_ON(!event))892 continue;893 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);894 }895}896 897static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)898{899 if (cci_pmu->model->write_counters)900 cci_pmu->model->write_counters(cci_pmu, mask);901 else902 __pmu_write_counters(cci_pmu, mask);903}904 905#ifdef CONFIG_ARM_CCI5xx_PMU906 907/*908 * CCI-500/CCI-550 has advanced power saving policies, which could gate the909 * clocks to the PMU counters, which makes the writes to them ineffective.910 * The only way to write to those counters is when the global counters911 * are enabled and the particular counter is enabled.912 *913 * So we do the following :914 *915 * 1) Disable all the PMU counters, saving their current state916 * 2) Enable the global PMU profiling, now that all counters are917 * disabled.918 *919 * For each counter to be programmed, repeat steps 3-7:920 *921 * 3) Write an invalid event code to the event control register for the922 counter, so that the counters are not modified.923 * 4) Enable the counter control for the counter.924 * 5) Set the counter value925 * 6) Disable the counter926 * 7) Restore the event in the target counter927 *928 * 8) Disable the global PMU.929 * 9) Restore the status of the rest of the counters.930 *931 * We choose an event which for CCI-5xx is guaranteed not to count.932 * We use the highest possible event code (0x1f) for the master interface 0.933 */934#define CCI5xx_INVALID_EVENT ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \935 (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT))936static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)937{938 int i;939 DECLARE_BITMAP(saved_mask, HW_CNTRS_MAX);940 941 bitmap_zero(saved_mask, cci_pmu->num_cntrs);942 pmu_save_counters(cci_pmu, saved_mask);943 944 /*945 * Now that all the counters are disabled, we can safely turn the PMU on,946 * without syncing the status of the counters947 */948 __cci_pmu_enable_nosync(cci_pmu);949 950 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {951 struct perf_event *event = cci_pmu->hw_events.events[i];952 953 if (WARN_ON(!event))954 continue;955 956 pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT);957 pmu_enable_counter(cci_pmu, i);958 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);959 pmu_disable_counter(cci_pmu, i);960 pmu_set_event(cci_pmu, i, event->hw.config_base);961 }962 963 __cci_pmu_disable(cci_pmu);964 965 pmu_restore_counters(cci_pmu, saved_mask);966}967 968#endif /* CONFIG_ARM_CCI5xx_PMU */969 970static u64 pmu_event_update(struct perf_event *event)971{972 struct hw_perf_event *hwc = &event->hw;973 u64 delta, prev_raw_count, new_raw_count;974 975 do {976 prev_raw_count = local64_read(&hwc->prev_count);977 new_raw_count = pmu_read_counter(event);978 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,979 new_raw_count) != prev_raw_count);980 981 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK;982 983 local64_add(delta, &event->count);984 985 return new_raw_count;986}987 988static void pmu_read(struct perf_event *event)989{990 pmu_event_update(event);991}992 993static void pmu_event_set_period(struct perf_event *event)994{995 struct hw_perf_event *hwc = &event->hw;996 /*997 * The CCI PMU counters have a period of 2^32. To account for the998 * possiblity of extreme interrupt latency we program for a period of999 * half that. Hopefully we can handle the interrupt before another 2^311000 * events occur and the counter overtakes its previous value.1001 */1002 u64 val = 1ULL << 31;1003 local64_set(&hwc->prev_count, val);1004 1005 /*1006 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose1007 * values needs to be sync-ed with the s/w state before the PMU is1008 * enabled.1009 * Mark this counter for sync.1010 */1011 hwc->state |= PERF_HES_ARCH;1012}1013 1014static irqreturn_t pmu_handle_irq(int irq_num, void *dev)1015{1016 struct cci_pmu *cci_pmu = dev;1017 struct cci_pmu_hw_events *events = &cci_pmu->hw_events;1018 int idx, handled = IRQ_NONE;1019 1020 raw_spin_lock(&events->pmu_lock);1021 1022 /* Disable the PMU while we walk through the counters */1023 __cci_pmu_disable(cci_pmu);1024 /*1025 * Iterate over counters and update the corresponding perf events.1026 * This should work regardless of whether we have per-counter overflow1027 * interrupt or a combined overflow interrupt.1028 */1029 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) {1030 struct perf_event *event = events->events[idx];1031 1032 if (!event)1033 continue;1034 1035 /* Did this counter overflow? */1036 if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) &1037 CCI_PMU_OVRFLW_FLAG))1038 continue;1039 1040 pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx,1041 CCI_PMU_OVRFLW);1042 1043 pmu_event_update(event);1044 pmu_event_set_period(event);1045 handled = IRQ_HANDLED;1046 }1047 1048 /* Enable the PMU and sync possibly overflowed counters */1049 __cci_pmu_enable_sync(cci_pmu);1050 raw_spin_unlock(&events->pmu_lock);1051 1052 return IRQ_RETVAL(handled);1053}1054 1055static int cci_pmu_get_hw(struct cci_pmu *cci_pmu)1056{1057 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq);1058 if (ret) {1059 pmu_free_irq(cci_pmu);1060 return ret;1061 }1062 return 0;1063}1064 1065static void cci_pmu_put_hw(struct cci_pmu *cci_pmu)1066{1067 pmu_free_irq(cci_pmu);1068}1069 1070static void hw_perf_event_destroy(struct perf_event *event)1071{1072 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);1073 atomic_t *active_events = &cci_pmu->active_events;1074 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex;1075 1076 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) {1077 cci_pmu_put_hw(cci_pmu);1078 mutex_unlock(reserve_mutex);1079 }1080}1081 1082static void cci_pmu_enable(struct pmu *pmu)1083{1084 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);1085 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;1086 bool enabled = !bitmap_empty(hw_events->used_mask, cci_pmu->num_cntrs);1087 unsigned long flags;1088 1089 if (!enabled)1090 return;1091 1092 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);1093 __cci_pmu_enable_sync(cci_pmu);1094 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);1095 1096}1097 1098static void cci_pmu_disable(struct pmu *pmu)1099{1100 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);1101 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;1102 unsigned long flags;1103 1104 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);1105 __cci_pmu_disable(cci_pmu);1106 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);1107}1108 1109/*1110 * Check if the idx represents a non-programmable counter.1111 * All the fixed event counters are mapped before the programmable1112 * counters.1113 */1114static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx)1115{1116 return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs);1117}1118 1119static void cci_pmu_start(struct perf_event *event, int pmu_flags)1120{1121 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);1122 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;1123 struct hw_perf_event *hwc = &event->hw;1124 int idx = hwc->idx;1125 unsigned long flags;1126 1127 /*1128 * To handle interrupt latency, we always reprogram the period1129 * regardless of PERF_EF_RELOAD.1130 */1131 if (pmu_flags & PERF_EF_RELOAD)1132 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));1133 1134 hwc->state = 0;1135 1136 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {1137 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);1138 return;1139 }1140 1141 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);1142 1143 /* Configure the counter unless you are counting a fixed event */1144 if (!pmu_fixed_hw_idx(cci_pmu, idx))1145 pmu_set_event(cci_pmu, idx, hwc->config_base);1146 1147 pmu_event_set_period(event);1148 pmu_enable_counter(cci_pmu, idx);1149 1150 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);1151}1152 1153static void cci_pmu_stop(struct perf_event *event, int pmu_flags)1154{1155 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);1156 struct hw_perf_event *hwc = &event->hw;1157 int idx = hwc->idx;1158 1159 if (hwc->state & PERF_HES_STOPPED)1160 return;1161 1162 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {1163 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);1164 return;1165 }1166 1167 /*1168 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See1169 * cci_pmu_start()1170 */1171 pmu_disable_counter(cci_pmu, idx);1172 pmu_event_update(event);1173 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;1174}1175 1176static int cci_pmu_add(struct perf_event *event, int flags)1177{1178 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);1179 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;1180 struct hw_perf_event *hwc = &event->hw;1181 int idx;1182 1183 /* If we don't have a space for the counter then finish early. */1184 idx = pmu_get_event_idx(hw_events, event);1185 if (idx < 0)1186 return idx;1187 1188 event->hw.idx = idx;1189 hw_events->events[idx] = event;1190 1191 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;1192 if (flags & PERF_EF_START)1193 cci_pmu_start(event, PERF_EF_RELOAD);1194 1195 /* Propagate our changes to the userspace mapping. */1196 perf_event_update_userpage(event);1197 1198 return 0;1199}1200 1201static void cci_pmu_del(struct perf_event *event, int flags)1202{1203 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);1204 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;1205 struct hw_perf_event *hwc = &event->hw;1206 int idx = hwc->idx;1207 1208 cci_pmu_stop(event, PERF_EF_UPDATE);1209 hw_events->events[idx] = NULL;1210 clear_bit(idx, hw_events->used_mask);1211 1212 perf_event_update_userpage(event);1213}1214 1215static int validate_event(struct pmu *cci_pmu,1216 struct cci_pmu_hw_events *hw_events,1217 struct perf_event *event)1218{1219 if (is_software_event(event))1220 return 1;1221 1222 /*1223 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The1224 * core perf code won't check that the pmu->ctx == leader->ctx1225 * until after pmu->event_init(event).1226 */1227 if (event->pmu != cci_pmu)1228 return 0;1229 1230 if (event->state < PERF_EVENT_STATE_OFF)1231 return 1;1232 1233 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)1234 return 1;1235 1236 return pmu_get_event_idx(hw_events, event) >= 0;1237}1238 1239static int validate_group(struct perf_event *event)1240{1241 struct perf_event *sibling, *leader = event->group_leader;1242 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);1243 unsigned long mask[BITS_TO_LONGS(HW_CNTRS_MAX)];1244 struct cci_pmu_hw_events fake_pmu = {1245 /*1246 * Initialise the fake PMU. We only need to populate the1247 * used_mask for the purposes of validation.1248 */1249 .used_mask = mask,1250 };1251 bitmap_zero(mask, cci_pmu->num_cntrs);1252 1253 if (!validate_event(event->pmu, &fake_pmu, leader))1254 return -EINVAL;1255 1256 for_each_sibling_event(sibling, leader) {1257 if (!validate_event(event->pmu, &fake_pmu, sibling))1258 return -EINVAL;1259 }1260 1261 if (!validate_event(event->pmu, &fake_pmu, event))1262 return -EINVAL;1263 1264 return 0;1265}1266 1267static int __hw_perf_event_init(struct perf_event *event)1268{1269 struct hw_perf_event *hwc = &event->hw;1270 int mapping;1271 1272 mapping = pmu_map_event(event);1273 1274 if (mapping < 0) {1275 pr_debug("event %x:%llx not supported\n", event->attr.type,1276 event->attr.config);1277 return mapping;1278 }1279 1280 /*1281 * We don't assign an index until we actually place the event onto1282 * hardware. Use -1 to signify that we haven't decided where to put it1283 * yet.1284 */1285 hwc->idx = -1;1286 hwc->config_base = 0;1287 hwc->config = 0;1288 hwc->event_base = 0;1289 1290 /*1291 * Store the event encoding into the config_base field.1292 */1293 hwc->config_base |= (unsigned long)mapping;1294 1295 if (event->group_leader != event) {1296 if (validate_group(event) != 0)1297 return -EINVAL;1298 }1299 1300 return 0;1301}1302 1303static int cci_pmu_event_init(struct perf_event *event)1304{1305 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);1306 atomic_t *active_events = &cci_pmu->active_events;1307 int err = 0;1308 1309 if (event->attr.type != event->pmu->type)1310 return -ENOENT;1311 1312 /* Shared by all CPUs, no meaningful state to sample */1313 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)1314 return -EOPNOTSUPP;1315 1316 /*1317 * Following the example set by other "uncore" PMUs, we accept any CPU1318 * and rewrite its affinity dynamically rather than having perf core1319 * handle cpu == -1 and pid == -1 for this case.1320 *1321 * The perf core will pin online CPUs for the duration of this call and1322 * the event being installed into its context, so the PMU's CPU can't1323 * change under our feet.1324 */1325 if (event->cpu < 0)1326 return -EINVAL;1327 event->cpu = cci_pmu->cpu;1328 1329 event->destroy = hw_perf_event_destroy;1330 if (!atomic_inc_not_zero(active_events)) {1331 mutex_lock(&cci_pmu->reserve_mutex);1332 if (atomic_read(active_events) == 0)1333 err = cci_pmu_get_hw(cci_pmu);1334 if (!err)1335 atomic_inc(active_events);1336 mutex_unlock(&cci_pmu->reserve_mutex);1337 }1338 if (err)1339 return err;1340 1341 err = __hw_perf_event_init(event);1342 if (err)1343 hw_perf_event_destroy(event);1344 1345 return err;1346}1347 1348static ssize_t pmu_cpumask_attr_show(struct device *dev,1349 struct device_attribute *attr, char *buf)1350{1351 struct pmu *pmu = dev_get_drvdata(dev);1352 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);1353 1354 return cpumap_print_to_pagebuf(true, buf, cpumask_of(cci_pmu->cpu));1355}1356 1357static struct device_attribute pmu_cpumask_attr =1358 __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL);1359 1360static struct attribute *pmu_attrs[] = {1361 &pmu_cpumask_attr.attr,1362 NULL,1363};1364 1365static const struct attribute_group pmu_attr_group = {1366 .attrs = pmu_attrs,1367};1368 1369static struct attribute_group pmu_format_attr_group = {1370 .name = "format",1371 .attrs = NULL, /* Filled in cci_pmu_init_attrs */1372};1373 1374static struct attribute_group pmu_event_attr_group = {1375 .name = "events",1376 .attrs = NULL, /* Filled in cci_pmu_init_attrs */1377};1378 1379static const struct attribute_group *pmu_attr_groups[] = {1380 &pmu_attr_group,1381 &pmu_format_attr_group,1382 &pmu_event_attr_group,1383 NULL1384};1385 1386static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)1387{1388 const struct cci_pmu_model *model = cci_pmu->model;1389 char *name = model->name;1390 u32 num_cntrs;1391 1392 if (WARN_ON(model->num_hw_cntrs > NUM_HW_CNTRS_MAX))1393 return -EINVAL;1394 if (WARN_ON(model->fixed_hw_cntrs > FIXED_HW_CNTRS_MAX))1395 return -EINVAL;1396 1397 pmu_event_attr_group.attrs = model->event_attrs;1398 pmu_format_attr_group.attrs = model->format_attrs;1399 1400 cci_pmu->pmu = (struct pmu) {1401 .module = THIS_MODULE,1402 .parent = &pdev->dev,1403 .name = cci_pmu->model->name,1404 .task_ctx_nr = perf_invalid_context,1405 .pmu_enable = cci_pmu_enable,1406 .pmu_disable = cci_pmu_disable,1407 .event_init = cci_pmu_event_init,1408 .add = cci_pmu_add,1409 .del = cci_pmu_del,1410 .start = cci_pmu_start,1411 .stop = cci_pmu_stop,1412 .read = pmu_read,1413 .attr_groups = pmu_attr_groups,1414 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,1415 };1416 1417 cci_pmu->plat_device = pdev;1418 num_cntrs = pmu_get_max_counters(cci_pmu);1419 if (num_cntrs > cci_pmu->model->num_hw_cntrs) {1420 dev_warn(&pdev->dev,1421 "PMU implements more counters(%d) than supported by"1422 " the model(%d), truncated.",1423 num_cntrs, cci_pmu->model->num_hw_cntrs);1424 num_cntrs = cci_pmu->model->num_hw_cntrs;1425 }1426 cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs;1427 1428 return perf_pmu_register(&cci_pmu->pmu, name, -1);1429}1430 1431static int cci_pmu_offline_cpu(unsigned int cpu)1432{1433 int target;1434 1435 if (!g_cci_pmu || cpu != g_cci_pmu->cpu)1436 return 0;1437 1438 target = cpumask_any_but(cpu_online_mask, cpu);1439 if (target >= nr_cpu_ids)1440 return 0;1441 1442 perf_pmu_migrate_context(&g_cci_pmu->pmu, cpu, target);1443 g_cci_pmu->cpu = target;1444 return 0;1445}1446 1447static __maybe_unused struct cci_pmu_model cci_pmu_models[] = {1448#ifdef CONFIG_ARM_CCI400_PMU1449 [CCI400_R0] = {1450 .name = "CCI_400",1451 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */1452 .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,1453 .cntr_size = SZ_4K,1454 .format_attrs = cci400_pmu_format_attrs,1455 .event_attrs = cci400_r0_pmu_event_attrs,1456 .event_ranges = {1457 [CCI_IF_SLAVE] = {1458 CCI400_R0_SLAVE_PORT_MIN_EV,1459 CCI400_R0_SLAVE_PORT_MAX_EV,1460 },1461 [CCI_IF_MASTER] = {1462 CCI400_R0_MASTER_PORT_MIN_EV,1463 CCI400_R0_MASTER_PORT_MAX_EV,1464 },1465 },1466 .validate_hw_event = cci400_validate_hw_event,1467 .get_event_idx = cci400_get_event_idx,1468 },1469 [CCI400_R1] = {1470 .name = "CCI_400_r1",1471 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */1472 .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,1473 .cntr_size = SZ_4K,1474 .format_attrs = cci400_pmu_format_attrs,1475 .event_attrs = cci400_r1_pmu_event_attrs,1476 .event_ranges = {1477 [CCI_IF_SLAVE] = {1478 CCI400_R1_SLAVE_PORT_MIN_EV,1479 CCI400_R1_SLAVE_PORT_MAX_EV,1480 },1481 [CCI_IF_MASTER] = {1482 CCI400_R1_MASTER_PORT_MIN_EV,1483 CCI400_R1_MASTER_PORT_MAX_EV,1484 },1485 },1486 .validate_hw_event = cci400_validate_hw_event,1487 .get_event_idx = cci400_get_event_idx,1488 },1489#endif1490#ifdef CONFIG_ARM_CCI5xx_PMU1491 [CCI500_R0] = {1492 .name = "CCI_500",1493 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,1494 .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,1495 .cntr_size = SZ_64K,1496 .format_attrs = cci5xx_pmu_format_attrs,1497 .event_attrs = cci5xx_pmu_event_attrs,1498 .event_ranges = {1499 [CCI_IF_SLAVE] = {1500 CCI5xx_SLAVE_PORT_MIN_EV,1501 CCI5xx_SLAVE_PORT_MAX_EV,1502 },1503 [CCI_IF_MASTER] = {1504 CCI5xx_MASTER_PORT_MIN_EV,1505 CCI5xx_MASTER_PORT_MAX_EV,1506 },1507 [CCI_IF_GLOBAL] = {1508 CCI5xx_GLOBAL_PORT_MIN_EV,1509 CCI5xx_GLOBAL_PORT_MAX_EV,1510 },1511 },1512 .validate_hw_event = cci500_validate_hw_event,1513 .write_counters = cci5xx_pmu_write_counters,1514 },1515 [CCI550_R0] = {1516 .name = "CCI_550",1517 .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,1518 .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,1519 .cntr_size = SZ_64K,1520 .format_attrs = cci5xx_pmu_format_attrs,1521 .event_attrs = cci5xx_pmu_event_attrs,1522 .event_ranges = {1523 [CCI_IF_SLAVE] = {1524 CCI5xx_SLAVE_PORT_MIN_EV,1525 CCI5xx_SLAVE_PORT_MAX_EV,1526 },1527 [CCI_IF_MASTER] = {1528 CCI5xx_MASTER_PORT_MIN_EV,1529 CCI5xx_MASTER_PORT_MAX_EV,1530 },1531 [CCI_IF_GLOBAL] = {1532 CCI5xx_GLOBAL_PORT_MIN_EV,1533 CCI5xx_GLOBAL_PORT_MAX_EV,1534 },1535 },1536 .validate_hw_event = cci550_validate_hw_event,1537 .write_counters = cci5xx_pmu_write_counters,1538 },1539#endif1540};1541 1542static const struct of_device_id arm_cci_pmu_matches[] = {1543#ifdef CONFIG_ARM_CCI400_PMU1544 {1545 .compatible = "arm,cci-400-pmu",1546 .data = NULL,1547 },1548 {1549 .compatible = "arm,cci-400-pmu,r0",1550 .data = &cci_pmu_models[CCI400_R0],1551 },1552 {1553 .compatible = "arm,cci-400-pmu,r1",1554 .data = &cci_pmu_models[CCI400_R1],1555 },1556#endif1557#ifdef CONFIG_ARM_CCI5xx_PMU1558 {1559 .compatible = "arm,cci-500-pmu,r0",1560 .data = &cci_pmu_models[CCI500_R0],1561 },1562 {1563 .compatible = "arm,cci-550-pmu,r0",1564 .data = &cci_pmu_models[CCI550_R0],1565 },1566#endif1567 {},1568};1569MODULE_DEVICE_TABLE(of, arm_cci_pmu_matches);1570 1571static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)1572{1573 int i;1574 1575 for (i = 0; i < nr_irqs; i++)1576 if (irq == irqs[i])1577 return true;1578 1579 return false;1580}1581 1582static struct cci_pmu *cci_pmu_alloc(struct device *dev)1583{1584 struct cci_pmu *cci_pmu;1585 const struct cci_pmu_model *model;1586 1587 /*1588 * All allocations are devm_* hence we don't have to free1589 * them explicitly on an error, as it would end up in driver1590 * detach.1591 */1592 cci_pmu = devm_kzalloc(dev, sizeof(*cci_pmu), GFP_KERNEL);1593 if (!cci_pmu)1594 return ERR_PTR(-ENOMEM);1595 1596 cci_pmu->ctrl_base = *(void __iomem **)dev->platform_data;1597 1598 model = of_device_get_match_data(dev);1599 if (!model) {1600 dev_warn(dev,1601 "DEPRECATED compatible property, requires secure access to CCI registers");1602 model = probe_cci_model(cci_pmu);1603 }1604 if (!model) {1605 dev_warn(dev, "CCI PMU version not supported\n");1606 return ERR_PTR(-ENODEV);1607 }1608 1609 cci_pmu->model = model;1610 cci_pmu->irqs = devm_kcalloc(dev, CCI_PMU_MAX_HW_CNTRS(model),1611 sizeof(*cci_pmu->irqs), GFP_KERNEL);1612 if (!cci_pmu->irqs)1613 return ERR_PTR(-ENOMEM);1614 cci_pmu->hw_events.events = devm_kcalloc(dev,1615 CCI_PMU_MAX_HW_CNTRS(model),1616 sizeof(*cci_pmu->hw_events.events),1617 GFP_KERNEL);1618 if (!cci_pmu->hw_events.events)1619 return ERR_PTR(-ENOMEM);1620 cci_pmu->hw_events.used_mask = devm_bitmap_zalloc(dev,1621 CCI_PMU_MAX_HW_CNTRS(model),1622 GFP_KERNEL);1623 if (!cci_pmu->hw_events.used_mask)1624 return ERR_PTR(-ENOMEM);1625 1626 return cci_pmu;1627}1628 1629static int cci_pmu_probe(struct platform_device *pdev)1630{1631 struct cci_pmu *cci_pmu;1632 int i, ret, irq;1633 1634 cci_pmu = cci_pmu_alloc(&pdev->dev);1635 if (IS_ERR(cci_pmu))1636 return PTR_ERR(cci_pmu);1637 1638 cci_pmu->base = devm_platform_ioremap_resource(pdev, 0);1639 if (IS_ERR(cci_pmu->base))1640 return -ENOMEM;1641 1642 /*1643 * CCI PMU has one overflow interrupt per counter; but some may be tied1644 * together to a common interrupt.1645 */1646 cci_pmu->nr_irqs = 0;1647 for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) {1648 irq = platform_get_irq(pdev, i);1649 if (irq < 0)1650 break;1651 1652 if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs))1653 continue;1654 1655 cci_pmu->irqs[cci_pmu->nr_irqs++] = irq;1656 }1657 1658 /*1659 * Ensure that the device tree has as many interrupts as the number1660 * of counters.1661 */1662 if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) {1663 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n",1664 i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model));1665 return -EINVAL;1666 }1667 1668 raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock);1669 mutex_init(&cci_pmu->reserve_mutex);1670 atomic_set(&cci_pmu->active_events, 0);1671 1672 cci_pmu->cpu = raw_smp_processor_id();1673 g_cci_pmu = cci_pmu;1674 cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,1675 "perf/arm/cci:online", NULL,1676 cci_pmu_offline_cpu);1677 1678 ret = cci_pmu_init(cci_pmu, pdev);1679 if (ret)1680 goto error_pmu_init;1681 1682 pr_info("ARM %s PMU driver probed", cci_pmu->model->name);1683 return 0;1684 1685error_pmu_init:1686 cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE);1687 g_cci_pmu = NULL;1688 return ret;1689}1690 1691static void cci_pmu_remove(struct platform_device *pdev)1692{1693 if (!g_cci_pmu)1694 return;1695 1696 cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE);1697 perf_pmu_unregister(&g_cci_pmu->pmu);1698 g_cci_pmu = NULL;1699}1700 1701static struct platform_driver cci_pmu_driver = {1702 .driver = {1703 .name = DRIVER_NAME,1704 .of_match_table = arm_cci_pmu_matches,1705 .suppress_bind_attrs = true,1706 },1707 .probe = cci_pmu_probe,1708 .remove_new = cci_pmu_remove,1709};1710 1711module_platform_driver(cci_pmu_driver);1712MODULE_LICENSE("GPL v2");1713MODULE_DESCRIPTION("ARM CCI PMU support");1714