brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02// Copyright (C) 2016-2020 Arm Limited3// CMN-600 Coherent Mesh Network PMU driver4 5#include <linux/acpi.h>6#include <linux/bitfield.h>7#include <linux/bitops.h>8#include <linux/debugfs.h>9#include <linux/interrupt.h>10#include <linux/io.h>11#include <linux/io-64-nonatomic-lo-hi.h>12#include <linux/kernel.h>13#include <linux/list.h>14#include <linux/module.h>15#include <linux/of.h>16#include <linux/perf_event.h>17#include <linux/platform_device.h>18#include <linux/slab.h>19#include <linux/sort.h>20 21/* Common register stuff */22#define CMN_NODE_INFO			0x000023#define CMN_NI_NODE_TYPE		GENMASK_ULL(15, 0)24#define CMN_NI_NODE_ID			GENMASK_ULL(31, 16)25#define CMN_NI_LOGICAL_ID		GENMASK_ULL(47, 32)26 27#define CMN_CHILD_INFO			0x008028#define CMN_CI_CHILD_COUNT		GENMASK_ULL(15, 0)29#define CMN_CI_CHILD_PTR_OFFSET		GENMASK_ULL(31, 16)30 31#define CMN_CHILD_NODE_ADDR		GENMASK(29, 0)32#define CMN_CHILD_NODE_EXTERNAL		BIT(31)33 34#define CMN_MAX_DIMENSION		1235#define CMN_MAX_XPS			(CMN_MAX_DIMENSION * CMN_MAX_DIMENSION)36#define CMN_MAX_DTMS			(CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4)37 38/* Currently XPs are the node type we can have most of; others top out at 128 */39#define CMN_MAX_NODES_PER_EVENT		CMN_MAX_XPS40 41/* The CFG node has various info besides the discovery tree */42#define CMN_CFGM_PERIPH_ID_01		0x000843#define CMN_CFGM_PID0_PART_0		GENMASK_ULL(7, 0)44#define CMN_CFGM_PID1_PART_1		GENMASK_ULL(35, 32)45#define CMN_CFGM_PERIPH_ID_23		0x001046#define CMN_CFGM_PID2_REVISION		GENMASK_ULL(7, 4)47 48#define CMN_CFGM_INFO_GLOBAL		0x090049#define CMN_INFO_MULTIPLE_DTM_EN	BIT_ULL(63)50#define CMN_INFO_RSP_VC_NUM		GENMASK_ULL(53, 52)51#define CMN_INFO_DAT_VC_NUM		GENMASK_ULL(51, 50)52#define CMN_INFO_DEVICE_ISO_ENABLE	BIT_ULL(44)53 54#define CMN_CFGM_INFO_GLOBAL_1		0x090855#define CMN_INFO_SNP_VC_NUM		GENMASK_ULL(3, 2)56#define CMN_INFO_REQ_VC_NUM		GENMASK_ULL(1, 0)57 58/* XPs also have some local topology info which has uses too */59#define CMN_MXP__CONNECT_INFO(p)	(0x0008 + 8 * (p))60#define CMN__CONNECT_INFO_DEVICE_TYPE	GENMASK_ULL(5, 0)61 62#define CMN_MAX_PORTS			663#define CI700_CONNECT_INFO_P2_5_OFFSET	0x1064 65/* PMU registers occupy the 3rd 4KB page of each node's region */66#define CMN_PMU_OFFSET			0x200067/* ...except when they don't :( */68#define CMN_S3_DTM_OFFSET		0xa00069#define CMN_S3_PMU_OFFSET		0xd90070 71/* For most nodes, this is all there is */72#define CMN_PMU_EVENT_SEL		0x00073#define CMN__PMU_CBUSY_SNTHROTTLE_SEL	GENMASK_ULL(44, 42)74#define CMN__PMU_SN_HOME_SEL		GENMASK_ULL(40, 39)75#define CMN__PMU_HBT_LBT_SEL		GENMASK_ULL(38, 37)76#define CMN__PMU_CLASS_OCCUP_ID		GENMASK_ULL(36, 35)77/* Technically this is 4 bits wide on DNs, but we only use 2 there anyway */78#define CMN__PMU_OCCUP1_ID		GENMASK_ULL(34, 32)79 80/* Some types are designed to coexist with another device in the same node */81#define CMN_CCLA_PMU_EVENT_SEL		0x00882#define CMN_HNP_PMU_EVENT_SEL		0x00883 84/* DTMs live in the PMU space of XP registers */85#define CMN_DTM_WPn(n)			(0x1A0 + (n) * 0x18)86#define CMN_DTM_WPn_CONFIG(n)		(CMN_DTM_WPn(n) + 0x00)87#define CMN_DTM_WPn_CONFIG_WP_CHN_NUM	GENMASK_ULL(20, 19)88#define CMN_DTM_WPn_CONFIG_WP_DEV_SEL2	GENMASK_ULL(18, 17)89#define CMN_DTM_WPn_CONFIG_WP_COMBINE	BIT(9)90#define CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE	BIT(8)91#define CMN600_WPn_CONFIG_WP_COMBINE	BIT(6)92#define CMN600_WPn_CONFIG_WP_EXCLUSIVE	BIT(5)93#define CMN_DTM_WPn_CONFIG_WP_GRP	GENMASK_ULL(5, 4)94#define CMN_DTM_WPn_CONFIG_WP_CHN_SEL	GENMASK_ULL(3, 1)95#define CMN_DTM_WPn_CONFIG_WP_DEV_SEL	BIT(0)96#define CMN_DTM_WPn_VAL(n)		(CMN_DTM_WPn(n) + 0x08)97#define CMN_DTM_WPn_MASK(n)		(CMN_DTM_WPn(n) + 0x10)98 99#define CMN_DTM_PMU_CONFIG		0x210100#define CMN__PMEVCNT0_INPUT_SEL		GENMASK_ULL(37, 32)101#define CMN__PMEVCNT0_INPUT_SEL_WP	0x00102#define CMN__PMEVCNT0_INPUT_SEL_XP	0x04103#define CMN__PMEVCNT0_INPUT_SEL_DEV	0x10104#define CMN__PMEVCNT0_GLOBAL_NUM	GENMASK_ULL(18, 16)105#define CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(n)	((n) * 4)106#define CMN__PMEVCNT_PAIRED(n)		BIT(4 + (n))107#define CMN__PMEVCNT23_COMBINED		BIT(2)108#define CMN__PMEVCNT01_COMBINED		BIT(1)109#define CMN_DTM_PMU_CONFIG_PMU_EN	BIT(0)110 111#define CMN_DTM_PMEVCNT			0x220112 113#define CMN_DTM_PMEVCNTSR		0x240114 115#define CMN650_DTM_UNIT_INFO		0x0910116#define CMN_DTM_UNIT_INFO		0x0960117#define CMN_DTM_UNIT_INFO_DTC_DOMAIN	GENMASK_ULL(1, 0)118 119#define CMN_DTM_NUM_COUNTERS		4120/* Want more local counters? Why not replicate the whole DTM! Ugh... */121#define CMN_DTM_OFFSET(n)		((n) * 0x200)122 123/* The DTC node is where the magic happens */124#define CMN_DT_DTC_CTL			0x0a00125#define CMN_DT_DTC_CTL_DT_EN		BIT(0)126#define CMN_DT_DTC_CTL_CG_DISABLE	BIT(10)127 128/* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */129#define _CMN_DT_CNT_REG(n)		((((n) / 2) * 4 + (n) % 2) * 4)130#define CMN_DT_PMEVCNT(dtc, n)		((dtc)->pmu_base + _CMN_DT_CNT_REG(n))131#define CMN_DT_PMCCNTR(dtc)		((dtc)->pmu_base + 0x40)132 133#define CMN_DT_PMEVCNTSR(dtc, n)	((dtc)->pmu_base + 0x50 + _CMN_DT_CNT_REG(n))134#define CMN_DT_PMCCNTRSR(dtc)		((dtc)->pmu_base + 0x90)135 136#define CMN_DT_PMCR(dtc)		((dtc)->pmu_base + 0x100)137#define CMN_DT_PMCR_PMU_EN		BIT(0)138#define CMN_DT_PMCR_CNTR_RST		BIT(5)139#define CMN_DT_PMCR_OVFL_INTR_EN	BIT(6)140 141#define CMN_DT_PMOVSR(dtc)		((dtc)->pmu_base + 0x118)142#define CMN_DT_PMOVSR_CLR(dtc)		((dtc)->pmu_base + 0x120)143 144#define CMN_DT_PMSSR(dtc)		((dtc)->pmu_base + 0x128)145#define CMN_DT_PMSSR_SS_STATUS(n)	BIT(n)146 147#define CMN_DT_PMSRR(dtc)		((dtc)->pmu_base + 0x130)148#define CMN_DT_PMSRR_SS_REQ		BIT(0)149 150#define CMN_DT_NUM_COUNTERS		8151#define CMN_MAX_DTCS			4152 153/*154 * Even in the worst case a DTC counter can't wrap in fewer than 2^42 cycles,155 * so throwing away one bit to make overflow handling easy is no big deal.156 */157#define CMN_COUNTER_INIT		0x80000000158/* Similarly for the 40-bit cycle counter */159#define CMN_CC_INIT			0x8000000000ULL160 161 162/* Event attributes */163#define CMN_CONFIG_TYPE			GENMASK_ULL(15, 0)164#define CMN_CONFIG_EVENTID		GENMASK_ULL(26, 16)165#define CMN_CONFIG_OCCUPID		GENMASK_ULL(30, 27)166#define CMN_CONFIG_BYNODEID		BIT_ULL(31)167#define CMN_CONFIG_NODEID		GENMASK_ULL(47, 32)168 169#define CMN_EVENT_TYPE(event)		FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config)170#define CMN_EVENT_EVENTID(event)	FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config)171#define CMN_EVENT_OCCUPID(event)	FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config)172#define CMN_EVENT_BYNODEID(event)	FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)173#define CMN_EVENT_NODEID(event)		FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)174 175#define CMN_CONFIG_WP_COMBINE		GENMASK_ULL(30, 27)176#define CMN_CONFIG_WP_DEV_SEL		GENMASK_ULL(50, 48)177#define CMN_CONFIG_WP_CHN_SEL		GENMASK_ULL(55, 51)178#define CMN_CONFIG_WP_GRP		GENMASK_ULL(57, 56)179#define CMN_CONFIG_WP_EXCLUSIVE		BIT_ULL(58)180#define CMN_CONFIG1_WP_VAL		GENMASK_ULL(63, 0)181#define CMN_CONFIG2_WP_MASK		GENMASK_ULL(63, 0)182 183#define CMN_EVENT_WP_COMBINE(event)	FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config)184#define CMN_EVENT_WP_DEV_SEL(event)	FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config)185#define CMN_EVENT_WP_CHN_SEL(event)	FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config)186#define CMN_EVENT_WP_GRP(event)		FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config)187#define CMN_EVENT_WP_EXCLUSIVE(event)	FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config)188#define CMN_EVENT_WP_VAL(event)		FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1)189#define CMN_EVENT_WP_MASK(event)	FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2)190 191/* Made-up event IDs for watchpoint direction */192#define CMN_WP_UP			0193#define CMN_WP_DOWN			2194 195 196/* Internal values for encoding event support */197enum cmn_model {198	CMN600 = 1,199	CMN650 = 2,200	CMN700 = 4,201	CI700 = 8,202	CMNS3 = 16,203	/* ...and then we can use bitmap tricks for commonality */204	CMN_ANY = -1,205	NOT_CMN600 = -2,206	CMN_650ON = CMN650 | CMN700 | CMNS3,207};208 209/* Actual part numbers and revision IDs defined by the hardware */210enum cmn_part {211	PART_CMN600 = 0x434,212	PART_CMN650 = 0x436,213	PART_CMN700 = 0x43c,214	PART_CI700 = 0x43a,215	PART_CMN_S3 = 0x43e,216};217 218/* CMN-600 r0px shouldn't exist in silicon, thankfully */219enum cmn_revision {220	REV_CMN600_R1P0,221	REV_CMN600_R1P1,222	REV_CMN600_R1P2,223	REV_CMN600_R1P3,224	REV_CMN600_R2P0,225	REV_CMN600_R3P0,226	REV_CMN600_R3P1,227	REV_CMN650_R0P0 = 0,228	REV_CMN650_R1P0,229	REV_CMN650_R1P1,230	REV_CMN650_R2P0,231	REV_CMN650_R1P2,232	REV_CMN700_R0P0 = 0,233	REV_CMN700_R1P0,234	REV_CMN700_R2P0,235	REV_CMN700_R3P0,236	REV_CI700_R0P0 = 0,237	REV_CI700_R1P0,238	REV_CI700_R2P0,239};240 241enum cmn_node_type {242	CMN_TYPE_INVALID,243	CMN_TYPE_DVM,244	CMN_TYPE_CFG,245	CMN_TYPE_DTC,246	CMN_TYPE_HNI,247	CMN_TYPE_HNF,248	CMN_TYPE_XP,249	CMN_TYPE_SBSX,250	CMN_TYPE_MPAM_S,251	CMN_TYPE_MPAM_NS,252	CMN_TYPE_RNI,253	CMN_TYPE_RND = 0xd,254	CMN_TYPE_RNSAM = 0xf,255	CMN_TYPE_MTSX,256	CMN_TYPE_HNP,257	CMN_TYPE_CXRA = 0x100,258	CMN_TYPE_CXHA,259	CMN_TYPE_CXLA,260	CMN_TYPE_CCRA,261	CMN_TYPE_CCHA,262	CMN_TYPE_CCLA,263	CMN_TYPE_CCLA_RNI,264	CMN_TYPE_HNS = 0x200,265	CMN_TYPE_HNS_MPAM_S,266	CMN_TYPE_HNS_MPAM_NS,267	CMN_TYPE_APB = 0x1000,268	/* Not a real node type */269	CMN_TYPE_WP = 0x7770270};271 272enum cmn_filter_select {273	SEL_NONE = -1,274	SEL_OCCUP1ID,275	SEL_CLASS_OCCUP_ID,276	SEL_CBUSY_SNTHROTTLE_SEL,277	SEL_HBT_LBT_SEL,278	SEL_SN_HOME_SEL,279	SEL_MAX280};281 282struct arm_cmn_node {283	void __iomem *pmu_base;284	u16 id, logid;285	enum cmn_node_type type;286 287	/* XP properties really, but replicated to children for convenience */288	u8 dtm;289	s8 dtc;290	u8 portid_bits:4;291	u8 deviceid_bits:4;292	/* DN/HN-F/CXHA */293	struct {294		u8 val : 4;295		u8 count : 4;296	} occupid[SEL_MAX];297	union {298		u8 event[4];299		__le32 event_sel;300		u16 event_w[4];301		__le64 event_sel_w;302	};303};304 305struct arm_cmn_dtm {306	void __iomem *base;307	u32 pmu_config_low;308	union {309		u8 input_sel[4];310		__le32 pmu_config_high;311	};312	s8 wp_event[4];313};314 315struct arm_cmn_dtc {316	void __iomem *base;317	void __iomem *pmu_base;318	int irq;319	s8 irq_friend;320	bool cc_active;321 322	struct perf_event *counters[CMN_DT_NUM_COUNTERS];323	struct perf_event *cycles;324};325 326#define CMN_STATE_DISABLED	BIT(0)327#define CMN_STATE_TXN		BIT(1)328 329struct arm_cmn {330	struct device *dev;331	void __iomem *base;332	unsigned int state;333 334	enum cmn_revision rev;335	enum cmn_part part;336	u8 mesh_x;337	u8 mesh_y;338	u16 num_xps;339	u16 num_dns;340	bool multi_dtm;341	u8 ports_used;342	struct {343		unsigned int rsp_vc_num : 2;344		unsigned int dat_vc_num : 2;345		unsigned int snp_vc_num : 2;346		unsigned int req_vc_num : 2;347	};348 349	struct arm_cmn_node *xps;350	struct arm_cmn_node *dns;351 352	struct arm_cmn_dtm *dtms;353	struct arm_cmn_dtc *dtc;354	unsigned int num_dtcs;355 356	int cpu;357	struct hlist_node cpuhp_node;358 359	struct pmu pmu;360	struct dentry *debug;361};362 363#define to_cmn(p)	container_of(p, struct arm_cmn, pmu)364 365static int arm_cmn_hp_state;366 367struct arm_cmn_nodeid {368	u8 port;369	u8 dev;370};371 372static int arm_cmn_xyidbits(const struct arm_cmn *cmn)373{374	return fls((cmn->mesh_x - 1) | (cmn->mesh_y - 1));375}376 377static struct arm_cmn_nodeid arm_cmn_nid(const struct arm_cmn_node *dn)378{379	struct arm_cmn_nodeid nid;380 381	nid.dev = dn->id & ((1U << dn->deviceid_bits) - 1);382	nid.port = (dn->id >> dn->deviceid_bits) & ((1U << dn->portid_bits) - 1);383	return nid;384}385 386static struct arm_cmn_node *arm_cmn_node_to_xp(const struct arm_cmn *cmn,387					       const struct arm_cmn_node *dn)388{389	int id = dn->id >> (dn->portid_bits + dn->deviceid_bits);390	int bits = arm_cmn_xyidbits(cmn);391	int x = id >> bits;392	int y = id & ((1U << bits) - 1);393 394	return cmn->xps + cmn->mesh_x * y + x;395}396static struct arm_cmn_node *arm_cmn_node(const struct arm_cmn *cmn,397					 enum cmn_node_type type)398{399	struct arm_cmn_node *dn;400 401	for (dn = cmn->dns; dn->type; dn++)402		if (dn->type == type)403			return dn;404	return NULL;405}406 407static enum cmn_model arm_cmn_model(const struct arm_cmn *cmn)408{409	switch (cmn->part) {410	case PART_CMN600:411		return CMN600;412	case PART_CMN650:413		return CMN650;414	case PART_CMN700:415		return CMN700;416	case PART_CI700:417		return CI700;418	case PART_CMN_S3:419		return CMNS3;420	default:421		return 0;422	};423}424 425static int arm_cmn_pmu_offset(const struct arm_cmn *cmn, const struct arm_cmn_node *dn)426{427	if (cmn->part == PART_CMN_S3) {428		if (dn->type == CMN_TYPE_XP)429			return CMN_S3_DTM_OFFSET;430		return CMN_S3_PMU_OFFSET;431	}432	return CMN_PMU_OFFSET;433}434 435static u32 arm_cmn_device_connect_info(const struct arm_cmn *cmn,436				       const struct arm_cmn_node *xp, int port)437{438	int offset = CMN_MXP__CONNECT_INFO(port) - arm_cmn_pmu_offset(cmn, xp);439 440	if (port >= 2) {441		if (cmn->part == PART_CMN600 || cmn->part == PART_CMN650)442			return 0;443		/*444		 * CI-700 may have extra ports, but still has the445		 * mesh_port_connect_info registers in the way.446		 */447		if (cmn->part == PART_CI700)448			offset += CI700_CONNECT_INFO_P2_5_OFFSET;449	}450 451	return readl_relaxed(xp->pmu_base + offset);452}453 454static struct dentry *arm_cmn_debugfs;455 456#ifdef CONFIG_DEBUG_FS457static const char *arm_cmn_device_type(u8 type)458{459	switch(FIELD_GET(CMN__CONNECT_INFO_DEVICE_TYPE, type)) {460		case 0x00: return "        |";461		case 0x01: return "  RN-I  |";462		case 0x02: return "  RN-D  |";463		case 0x04: return " RN-F_B |";464		case 0x05: return "RN-F_B_E|";465		case 0x06: return " RN-F_A |";466		case 0x07: return "RN-F_A_E|";467		case 0x08: return "  HN-T  |";468		case 0x09: return "  HN-I  |";469		case 0x0a: return "  HN-D  |";470		case 0x0b: return "  HN-P  |";471		case 0x0c: return "  SN-F  |";472		case 0x0d: return "  SBSX  |";473		case 0x0e: return "  HN-F  |";474		case 0x0f: return " SN-F_E |";475		case 0x10: return " SN-F_D |";476		case 0x11: return "  CXHA  |";477		case 0x12: return "  CXRA  |";478		case 0x13: return "  CXRH  |";479		case 0x14: return " RN-F_D |";480		case 0x15: return "RN-F_D_E|";481		case 0x16: return " RN-F_C |";482		case 0x17: return "RN-F_C_E|";483		case 0x18: return " RN-F_E |";484		case 0x19: return "RN-F_E_E|";485		case 0x1a: return "  HN-S  |";486		case 0x1b: return "  LCN   |";487		case 0x1c: return "  MTSX  |";488		case 0x1d: return "  HN-V  |";489		case 0x1e: return "  CCG   |";490		case 0x20: return " RN-F_F |";491		case 0x21: return "RN-F_F_E|";492		case 0x22: return " SN-F_F |";493		default:   return "  ????  |";494	}495}496 497static void arm_cmn_show_logid(struct seq_file *s, const struct arm_cmn_node *xp, int p, int d)498{499	struct arm_cmn *cmn = s->private;500	struct arm_cmn_node *dn;501	u16 id = xp->id | d | (p << xp->deviceid_bits);502 503	for (dn = cmn->dns; dn->type; dn++) {504		int pad = dn->logid < 10;505 506		if (dn->type == CMN_TYPE_XP)507			continue;508		/* Ignore the extra components that will overlap on some ports */509		if (dn->type < CMN_TYPE_HNI)510			continue;511 512		if (dn->id != id)513			continue;514 515		seq_printf(s, " %*c#%-*d  |", pad + 1, ' ', 3 - pad, dn->logid);516		return;517	}518	seq_puts(s, "        |");519}520 521static int arm_cmn_map_show(struct seq_file *s, void *data)522{523	struct arm_cmn *cmn = s->private;524	int x, y, p, pmax = fls(cmn->ports_used);525 526	seq_puts(s, "     X");527	for (x = 0; x < cmn->mesh_x; x++)528		seq_printf(s, "    %-2d   ", x);529	seq_puts(s, "\nY P D+");530	y = cmn->mesh_y;531	while (y--) {532		int xp_base = cmn->mesh_x * y;533		struct arm_cmn_node *xp = cmn->xps + xp_base;534		u8 port[CMN_MAX_PORTS][CMN_MAX_DIMENSION];535 536		for (x = 0; x < cmn->mesh_x; x++)537			seq_puts(s, "--------+");538 539		seq_printf(s, "\n%-2d   |", y);540		for (x = 0; x < cmn->mesh_x; x++) {541			for (p = 0; p < CMN_MAX_PORTS; p++)542				port[p][x] = arm_cmn_device_connect_info(cmn, xp + x, p);543			seq_printf(s, " XP #%-3d|", xp_base + x);544		}545 546		seq_puts(s, "\n     |");547		for (x = 0; x < cmn->mesh_x; x++) {548			s8 dtc = xp[x].dtc;549 550			if (dtc < 0)551				seq_puts(s, " DTC ?? |");552			else553				seq_printf(s, " DTC %d  |", dtc);554		}555		seq_puts(s, "\n     |");556		for (x = 0; x < cmn->mesh_x; x++)557			seq_puts(s, "........|");558 559		for (p = 0; p < pmax; p++) {560			seq_printf(s, "\n  %d  |", p);561			for (x = 0; x < cmn->mesh_x; x++)562				seq_puts(s, arm_cmn_device_type(port[p][x]));563			seq_puts(s, "\n    0|");564			for (x = 0; x < cmn->mesh_x; x++)565				arm_cmn_show_logid(s, xp + x, p, 0);566			seq_puts(s, "\n    1|");567			for (x = 0; x < cmn->mesh_x; x++)568				arm_cmn_show_logid(s, xp + x, p, 1);569		}570		seq_puts(s, "\n-----+");571	}572	for (x = 0; x < cmn->mesh_x; x++)573		seq_puts(s, "--------+");574	seq_puts(s, "\n");575	return 0;576}577DEFINE_SHOW_ATTRIBUTE(arm_cmn_map);578 579static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id)580{581	const char *name  = "map";582 583	if (id > 0)584		name = devm_kasprintf(cmn->dev, GFP_KERNEL, "map_%d", id);585	if (!name)586		return;587 588	cmn->debug = debugfs_create_file(name, 0444, arm_cmn_debugfs, cmn, &arm_cmn_map_fops);589}590#else591static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) {}592#endif593 594struct arm_cmn_hw_event {595	struct arm_cmn_node *dn;596	u64 dtm_idx[DIV_ROUND_UP(CMN_MAX_NODES_PER_EVENT * 2, 64)];597	s8 dtc_idx[CMN_MAX_DTCS];598	u8 num_dns;599	u8 dtm_offset;600 601	/*602	 * WP config registers are divided to UP and DOWN events. We need to603	 * keep to track only one of them.604	 */605	DECLARE_BITMAP(wp_idx, CMN_MAX_XPS);606 607	bool wide_sel;608	enum cmn_filter_select filter_sel;609};610static_assert(sizeof(struct arm_cmn_hw_event) <= offsetof(struct hw_perf_event, target));611 612#define for_each_hw_dn(hw, dn, i) \613	for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++)614 615/* @i is the DTC number, @idx is the counter index on that DTC */616#define for_each_hw_dtc_idx(hw, i, idx) \617	for (int i = 0, idx; i < CMN_MAX_DTCS; i++) if ((idx = hw->dtc_idx[i]) >= 0)618 619static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event)620{621	return (struct arm_cmn_hw_event *)&event->hw;622}623 624static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val)625{626	x[pos / 32] |= (u64)val << ((pos % 32) * 2);627}628 629static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos)630{631	return (x[pos / 32] >> ((pos % 32) * 2)) & 3;632}633 634static void arm_cmn_set_wp_idx(unsigned long *wp_idx, unsigned int pos, bool val)635{636	if (val)637		set_bit(pos, wp_idx);638}639 640static unsigned int arm_cmn_get_wp_idx(unsigned long *wp_idx, unsigned int pos)641{642	return test_bit(pos, wp_idx);643}644 645struct arm_cmn_event_attr {646	struct device_attribute attr;647	enum cmn_model model;648	enum cmn_node_type type;649	enum cmn_filter_select fsel;650	u16 eventid;651	u8 occupid;652};653 654struct arm_cmn_format_attr {655	struct device_attribute attr;656	u64 field;657	int config;658};659 660#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _occupid, _fsel)\661	(&((struct arm_cmn_event_attr[]) {{				\662		.attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL),	\663		.model = _model,					\664		.type = _type,						\665		.eventid = _eventid,					\666		.occupid = _occupid,					\667		.fsel = _fsel,						\668	}})[0].attr.attr)669#define CMN_EVENT_ATTR(_model, _name, _type, _eventid)			\670	_CMN_EVENT_ATTR(_model, _name, _type, _eventid, 0, SEL_NONE)671 672static ssize_t arm_cmn_event_show(struct device *dev,673				  struct device_attribute *attr, char *buf)674{675	struct arm_cmn_event_attr *eattr;676 677	eattr = container_of(attr, typeof(*eattr), attr);678 679	if (eattr->type == CMN_TYPE_DTC)680		return sysfs_emit(buf, "type=0x%x\n", eattr->type);681 682	if (eattr->type == CMN_TYPE_WP)683		return sysfs_emit(buf,684				  "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n",685				  eattr->type, eattr->eventid);686 687	if (eattr->fsel > SEL_NONE)688		return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n",689				  eattr->type, eattr->eventid, eattr->occupid);690 691	return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type,692			  eattr->eventid);693}694 695static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj,696					     struct attribute *attr,697					     int unused)698{699	struct device *dev = kobj_to_dev(kobj);700	struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));701	struct arm_cmn_event_attr *eattr;702	enum cmn_node_type type;703	u16 eventid;704 705	eattr = container_of(attr, typeof(*eattr), attr.attr);706 707	if (!(eattr->model & arm_cmn_model(cmn)))708		return 0;709 710	type = eattr->type;711	eventid = eattr->eventid;712 713	/* Watchpoints aren't nodes, so avoid confusion */714	if (type == CMN_TYPE_WP)715		return attr->mode;716 717	/* Hide XP events for unused interfaces/channels */718	if (type == CMN_TYPE_XP) {719		unsigned int intf = (eventid >> 2) & 7;720		unsigned int chan = eventid >> 5;721 722		if ((intf & 4) && !(cmn->ports_used & BIT(intf & 3)))723			return 0;724 725		if (chan == 4 && cmn->part == PART_CMN600)726			return 0;727 728		if ((chan == 5 && cmn->rsp_vc_num < 2) ||729		    (chan == 6 && cmn->dat_vc_num < 2) ||730		    (chan == 7 && cmn->snp_vc_num < 2) ||731		    (chan == 8 && cmn->req_vc_num < 2))732			return 0;733	}734 735	/* Revision-specific differences */736	if (cmn->part == PART_CMN600) {737		if (cmn->rev < REV_CMN600_R1P3) {738			if (type == CMN_TYPE_CXRA && eventid > 0x10)739				return 0;740		}741		if (cmn->rev < REV_CMN600_R1P2) {742			if (type == CMN_TYPE_HNF && eventid == 0x1b)743				return 0;744			if (type == CMN_TYPE_CXRA || type == CMN_TYPE_CXHA)745				return 0;746		}747	} else if (cmn->part == PART_CMN650) {748		if (cmn->rev < REV_CMN650_R2P0 || cmn->rev == REV_CMN650_R1P2) {749			if (type == CMN_TYPE_HNF && eventid > 0x22)750				return 0;751			if (type == CMN_TYPE_SBSX && eventid == 0x17)752				return 0;753			if (type == CMN_TYPE_RNI && eventid > 0x10)754				return 0;755		}756	} else if (cmn->part == PART_CMN700) {757		if (cmn->rev < REV_CMN700_R2P0) {758			if (type == CMN_TYPE_HNF && eventid > 0x2c)759				return 0;760			if (type == CMN_TYPE_CCHA && eventid > 0x74)761				return 0;762			if (type == CMN_TYPE_CCLA && eventid > 0x27)763				return 0;764		}765		if (cmn->rev < REV_CMN700_R1P0) {766			if (type == CMN_TYPE_HNF && eventid > 0x2b)767				return 0;768		}769	}770 771	if (!arm_cmn_node(cmn, type))772		return 0;773 774	return attr->mode;775}776 777#define _CMN_EVENT_DVM(_model, _name, _event, _occup, _fsel)	\778	_CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, _occup, _fsel)779#define CMN_EVENT_DTC(_name)					\780	CMN_EVENT_ATTR(CMN_ANY, dtc_##_name, CMN_TYPE_DTC, 0)781#define CMN_EVENT_HNF(_model, _name, _event)			\782	CMN_EVENT_ATTR(_model, hnf_##_name, CMN_TYPE_HNF, _event)783#define CMN_EVENT_HNI(_name, _event)				\784	CMN_EVENT_ATTR(CMN_ANY, hni_##_name, CMN_TYPE_HNI, _event)785#define CMN_EVENT_HNP(_name, _event)				\786	CMN_EVENT_ATTR(CMN_ANY, hnp_##_name, CMN_TYPE_HNP, _event)787#define __CMN_EVENT_XP(_name, _event)				\788	CMN_EVENT_ATTR(CMN_ANY, mxp_##_name, CMN_TYPE_XP, _event)789#define CMN_EVENT_SBSX(_model, _name, _event)			\790	CMN_EVENT_ATTR(_model, sbsx_##_name, CMN_TYPE_SBSX, _event)791#define CMN_EVENT_RNID(_model, _name, _event)			\792	CMN_EVENT_ATTR(_model, rnid_##_name, CMN_TYPE_RNI, _event)793#define CMN_EVENT_MTSX(_name, _event)				\794	CMN_EVENT_ATTR(CMN_ANY, mtsx_##_name, CMN_TYPE_MTSX, _event)795#define CMN_EVENT_CXRA(_model, _name, _event)				\796	CMN_EVENT_ATTR(_model, cxra_##_name, CMN_TYPE_CXRA, _event)797#define CMN_EVENT_CXHA(_name, _event)				\798	CMN_EVENT_ATTR(CMN_ANY, cxha_##_name, CMN_TYPE_CXHA, _event)799#define CMN_EVENT_CCRA(_name, _event)				\800	CMN_EVENT_ATTR(CMN_ANY, ccra_##_name, CMN_TYPE_CCRA, _event)801#define CMN_EVENT_CCHA(_model, _name, _event)				\802	CMN_EVENT_ATTR(_model, ccha_##_name, CMN_TYPE_CCHA, _event)803#define CMN_EVENT_CCLA(_name, _event)				\804	CMN_EVENT_ATTR(CMN_ANY, ccla_##_name, CMN_TYPE_CCLA, _event)805#define CMN_EVENT_CCLA_RNI(_name, _event)				\806	CMN_EVENT_ATTR(CMN_ANY, ccla_rni_##_name, CMN_TYPE_CCLA_RNI, _event)807#define CMN_EVENT_HNS(_name, _event)				\808	CMN_EVENT_ATTR(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event)809 810#define CMN_EVENT_DVM(_model, _name, _event)			\811	_CMN_EVENT_DVM(_model, _name, _event, 0, SEL_NONE)812#define CMN_EVENT_DVM_OCC(_model, _name, _event)			\813	_CMN_EVENT_DVM(_model, _name##_all, _event, 0, SEL_OCCUP1ID),	\814	_CMN_EVENT_DVM(_model, _name##_dvmop, _event, 1, SEL_OCCUP1ID),	\815	_CMN_EVENT_DVM(_model, _name##_dvmsync, _event, 2, SEL_OCCUP1ID)816 817#define CMN_EVENT_HN_OCC(_model, _name, _type, _event)		\818	_CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_OCCUP1ID), \819	_CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 1, SEL_OCCUP1ID), \820	_CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 2, SEL_OCCUP1ID), \821	_CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, 3, SEL_OCCUP1ID), \822	_CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, 4, SEL_OCCUP1ID)823#define CMN_EVENT_HN_CLS(_model, _name, _type, _event)			\824	_CMN_EVENT_ATTR(_model, _name##_class0, _type, _event, 0, SEL_CLASS_OCCUP_ID), \825	_CMN_EVENT_ATTR(_model, _name##_class1, _type, _event, 1, SEL_CLASS_OCCUP_ID), \826	_CMN_EVENT_ATTR(_model, _name##_class2, _type, _event, 2, SEL_CLASS_OCCUP_ID), \827	_CMN_EVENT_ATTR(_model, _name##_class3, _type, _event, 3, SEL_CLASS_OCCUP_ID)828#define CMN_EVENT_HN_SNT(_model, _name, _type, _event)			\829	_CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_CBUSY_SNTHROTTLE_SEL), \830	_CMN_EVENT_ATTR(_model, _name##_group0_read, _type, _event, 1, SEL_CBUSY_SNTHROTTLE_SEL), \831	_CMN_EVENT_ATTR(_model, _name##_group0_write, _type, _event, 2, SEL_CBUSY_SNTHROTTLE_SEL), \832	_CMN_EVENT_ATTR(_model, _name##_group1_read, _type, _event, 3, SEL_CBUSY_SNTHROTTLE_SEL), \833	_CMN_EVENT_ATTR(_model, _name##_group1_write, _type, _event, 4, SEL_CBUSY_SNTHROTTLE_SEL), \834	_CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 5, SEL_CBUSY_SNTHROTTLE_SEL), \835	_CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 6, SEL_CBUSY_SNTHROTTLE_SEL)836 837#define CMN_EVENT_HNF_OCC(_model, _name, _event)			\838	CMN_EVENT_HN_OCC(_model, hnf_##_name, CMN_TYPE_HNF, _event)839#define CMN_EVENT_HNF_CLS(_model, _name, _event)			\840	CMN_EVENT_HN_CLS(_model, hnf_##_name, CMN_TYPE_HNF, _event)841#define CMN_EVENT_HNF_SNT(_model, _name, _event)			\842	CMN_EVENT_HN_SNT(_model, hnf_##_name, CMN_TYPE_HNF, _event)843 844#define CMN_EVENT_HNS_OCC(_name, _event)				\845	CMN_EVENT_HN_OCC(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event),	\846	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, 5, SEL_OCCUP1ID), \847	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 6, SEL_OCCUP1ID), \848	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 7, SEL_OCCUP1ID)849#define CMN_EVENT_HNS_CLS( _name, _event)				\850	CMN_EVENT_HN_CLS(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event)851#define CMN_EVENT_HNS_SNT(_name, _event)				\852	CMN_EVENT_HN_SNT(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event)853#define CMN_EVENT_HNS_HBT(_name, _event)				\854	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_HBT_LBT_SEL), \855	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 1, SEL_HBT_LBT_SEL), \856	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 2, SEL_HBT_LBT_SEL)857#define CMN_EVENT_HNS_SNH(_name, _event)				\858	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_SN_HOME_SEL), \859	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_sn, CMN_TYPE_HNS, _event, 1, SEL_SN_HOME_SEL), \860	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_home, CMN_TYPE_HNS, _event, 2, SEL_SN_HOME_SEL)861 862#define _CMN_EVENT_XP_MESH(_name, _event)			\863	__CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)),		\864	__CMN_EVENT_XP(w_##_name, (_event) | (1 << 2)),		\865	__CMN_EVENT_XP(n_##_name, (_event) | (2 << 2)),		\866	__CMN_EVENT_XP(s_##_name, (_event) | (3 << 2))867 868#define _CMN_EVENT_XP_PORT(_name, _event)			\869	__CMN_EVENT_XP(p0_##_name, (_event) | (4 << 2)),	\870	__CMN_EVENT_XP(p1_##_name, (_event) | (5 << 2)),	\871	__CMN_EVENT_XP(p2_##_name, (_event) | (6 << 2)),	\872	__CMN_EVENT_XP(p3_##_name, (_event) | (7 << 2))873 874#define _CMN_EVENT_XP(_name, _event)				\875	_CMN_EVENT_XP_MESH(_name, _event),			\876	_CMN_EVENT_XP_PORT(_name, _event)877 878/* Good thing there are only 3 fundamental XP events... */879#define CMN_EVENT_XP(_name, _event)				\880	_CMN_EVENT_XP(req_##_name, (_event) | (0 << 5)),	\881	_CMN_EVENT_XP(rsp_##_name, (_event) | (1 << 5)),	\882	_CMN_EVENT_XP(snp_##_name, (_event) | (2 << 5)),	\883	_CMN_EVENT_XP(dat_##_name, (_event) | (3 << 5)),	\884	_CMN_EVENT_XP(pub_##_name, (_event) | (4 << 5)),	\885	_CMN_EVENT_XP(rsp2_##_name, (_event) | (5 << 5)),	\886	_CMN_EVENT_XP(dat2_##_name, (_event) | (6 << 5)),	\887	_CMN_EVENT_XP(snp2_##_name, (_event) | (7 << 5)),	\888	_CMN_EVENT_XP(req2_##_name, (_event) | (8 << 5))889 890#define CMN_EVENT_XP_DAT(_name, _event)				\891	_CMN_EVENT_XP_PORT(dat_##_name, (_event) | (3 << 5)),	\892	_CMN_EVENT_XP_PORT(dat2_##_name, (_event) | (6 << 5))893 894 895static struct attribute *arm_cmn_event_attrs[] = {896	CMN_EVENT_DTC(cycles),897 898	/*899	 * DVM node events conflict with HN-I events in the equivalent PMU900	 * slot, but our lazy short-cut of using the DTM counter index for901	 * the PMU index as well happens to avoid that by construction.902	 */903	CMN_EVENT_DVM(CMN600, rxreq_dvmop,		0x01),904	CMN_EVENT_DVM(CMN600, rxreq_dvmsync,		0x02),905	CMN_EVENT_DVM(CMN600, rxreq_dvmop_vmid_filtered, 0x03),906	CMN_EVENT_DVM(CMN600, rxreq_retried,		0x04),907	CMN_EVENT_DVM_OCC(CMN600, rxreq_trk_occupancy,	0x05),908	CMN_EVENT_DVM(NOT_CMN600, dvmop_tlbi,		0x01),909	CMN_EVENT_DVM(NOT_CMN600, dvmop_bpi,		0x02),910	CMN_EVENT_DVM(NOT_CMN600, dvmop_pici,		0x03),911	CMN_EVENT_DVM(NOT_CMN600, dvmop_vici,		0x04),912	CMN_EVENT_DVM(NOT_CMN600, dvmsync,		0x05),913	CMN_EVENT_DVM(NOT_CMN600, vmid_filtered,	0x06),914	CMN_EVENT_DVM(NOT_CMN600, rndop_filtered,	0x07),915	CMN_EVENT_DVM(NOT_CMN600, retry,		0x08),916	CMN_EVENT_DVM(NOT_CMN600, txsnp_flitv,		0x09),917	CMN_EVENT_DVM(NOT_CMN600, txsnp_stall,		0x0a),918	CMN_EVENT_DVM(NOT_CMN600, trkfull,		0x0b),919	CMN_EVENT_DVM_OCC(NOT_CMN600, trk_occupancy,	0x0c),920	CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_cxha,	0x0d),921	CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_pdn,	0x0e),922	CMN_EVENT_DVM(CMN700, trk_alloc,		0x0f),923	CMN_EVENT_DVM(CMN700, trk_cxha_alloc,		0x10),924	CMN_EVENT_DVM(CMN700, trk_pdn_alloc,		0x11),925	CMN_EVENT_DVM(CMN700, txsnp_stall_limit,	0x12),926	CMN_EVENT_DVM(CMN700, rxsnp_stall_starv,	0x13),927	CMN_EVENT_DVM(CMN700, txsnp_sync_stall_op,	0x14),928 929	CMN_EVENT_HNF(CMN_ANY, cache_miss,		0x01),930	CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access,	0x02),931	CMN_EVENT_HNF(CMN_ANY, cache_fill,		0x03),932	CMN_EVENT_HNF(CMN_ANY, pocq_retry,		0x04),933	CMN_EVENT_HNF(CMN_ANY, pocq_reqs_recvd,		0x05),934	CMN_EVENT_HNF(CMN_ANY, sf_hit,			0x06),935	CMN_EVENT_HNF(CMN_ANY, sf_evictions,		0x07),936	CMN_EVENT_HNF(CMN_ANY, dir_snoops_sent,		0x08),937	CMN_EVENT_HNF(CMN_ANY, brd_snoops_sent,		0x09),938	CMN_EVENT_HNF(CMN_ANY, slc_eviction,		0x0a),939	CMN_EVENT_HNF(CMN_ANY, slc_fill_invalid_way,	0x0b),940	CMN_EVENT_HNF(CMN_ANY, mc_retries,		0x0c),941	CMN_EVENT_HNF(CMN_ANY, mc_reqs,			0x0d),942	CMN_EVENT_HNF(CMN_ANY, qos_hh_retry,		0x0e),943	CMN_EVENT_HNF_OCC(CMN_ANY, qos_pocq_occupancy,	0x0f),944	CMN_EVENT_HNF(CMN_ANY, pocq_addrhaz,		0x10),945	CMN_EVENT_HNF(CMN_ANY, pocq_atomic_addrhaz,	0x11),946	CMN_EVENT_HNF(CMN_ANY, ld_st_swp_adq_full,	0x12),947	CMN_EVENT_HNF(CMN_ANY, cmp_adq_full,		0x13),948	CMN_EVENT_HNF(CMN_ANY, txdat_stall,		0x14),949	CMN_EVENT_HNF(CMN_ANY, txrsp_stall,		0x15),950	CMN_EVENT_HNF(CMN_ANY, seq_full,		0x16),951	CMN_EVENT_HNF(CMN_ANY, seq_hit,			0x17),952	CMN_EVENT_HNF(CMN_ANY, snp_sent,		0x18),953	CMN_EVENT_HNF(CMN_ANY, sfbi_dir_snp_sent,	0x19),954	CMN_EVENT_HNF(CMN_ANY, sfbi_brd_snp_sent,	0x1a),955	CMN_EVENT_HNF(CMN_ANY, snp_sent_untrk,		0x1b),956	CMN_EVENT_HNF(CMN_ANY, intv_dirty,		0x1c),957	CMN_EVENT_HNF(CMN_ANY, stash_snp_sent,		0x1d),958	CMN_EVENT_HNF(CMN_ANY, stash_data_pull,		0x1e),959	CMN_EVENT_HNF(CMN_ANY, snp_fwded,		0x1f),960	CMN_EVENT_HNF(NOT_CMN600, atomic_fwd,		0x20),961	CMN_EVENT_HNF(NOT_CMN600, mpam_hardlim,		0x21),962	CMN_EVENT_HNF(NOT_CMN600, mpam_softlim,		0x22),963	CMN_EVENT_HNF(CMN_650ON, snp_sent_cluster,	0x23),964	CMN_EVENT_HNF(CMN_650ON, sf_imprecise_evict,	0x24),965	CMN_EVENT_HNF(CMN_650ON, sf_evict_shared_line,	0x25),966	CMN_EVENT_HNF_CLS(CMN700, pocq_class_occup,	0x26),967	CMN_EVENT_HNF_CLS(CMN700, pocq_class_retry,	0x27),968	CMN_EVENT_HNF_CLS(CMN700, class_mc_reqs,	0x28),969	CMN_EVENT_HNF_CLS(CMN700, class_cgnt_cmin,	0x29),970	CMN_EVENT_HNF_SNT(CMN700, sn_throttle,		0x2a),971	CMN_EVENT_HNF_SNT(CMN700, sn_throttle_min,	0x2b),972	CMN_EVENT_HNF(CMN700, sf_precise_to_imprecise,	0x2c),973	CMN_EVENT_HNF(CMN700, snp_intv_cln,		0x2d),974	CMN_EVENT_HNF(CMN700, nc_excl,			0x2e),975	CMN_EVENT_HNF(CMN700, excl_mon_ovfl,		0x2f),976 977	CMN_EVENT_HNI(rrt_rd_occ_cnt_ovfl,		0x20),978	CMN_EVENT_HNI(rrt_wr_occ_cnt_ovfl,		0x21),979	CMN_EVENT_HNI(rdt_rd_occ_cnt_ovfl,		0x22),980	CMN_EVENT_HNI(rdt_wr_occ_cnt_ovfl,		0x23),981	CMN_EVENT_HNI(wdb_occ_cnt_ovfl,			0x24),982	CMN_EVENT_HNI(rrt_rd_alloc,			0x25),983	CMN_EVENT_HNI(rrt_wr_alloc,			0x26),984	CMN_EVENT_HNI(rdt_rd_alloc,			0x27),985	CMN_EVENT_HNI(rdt_wr_alloc,			0x28),986	CMN_EVENT_HNI(wdb_alloc,			0x29),987	CMN_EVENT_HNI(txrsp_retryack,			0x2a),988	CMN_EVENT_HNI(arvalid_no_arready,		0x2b),989	CMN_EVENT_HNI(arready_no_arvalid,		0x2c),990	CMN_EVENT_HNI(awvalid_no_awready,		0x2d),991	CMN_EVENT_HNI(awready_no_awvalid,		0x2e),992	CMN_EVENT_HNI(wvalid_no_wready,			0x2f),993	CMN_EVENT_HNI(txdat_stall,			0x30),994	CMN_EVENT_HNI(nonpcie_serialization,		0x31),995	CMN_EVENT_HNI(pcie_serialization,		0x32),996 997	/*998	 * HN-P events squat on top of the HN-I similarly to DVM events, except999	 * for being crammed into the same physical node as well. And of course1000	 * where would the fun be if the same events were in the same order...1001	 */1002	CMN_EVENT_HNP(rrt_wr_occ_cnt_ovfl,		0x01),1003	CMN_EVENT_HNP(rdt_wr_occ_cnt_ovfl,		0x02),1004	CMN_EVENT_HNP(wdb_occ_cnt_ovfl,			0x03),1005	CMN_EVENT_HNP(rrt_wr_alloc,			0x04),1006	CMN_EVENT_HNP(rdt_wr_alloc,			0x05),1007	CMN_EVENT_HNP(wdb_alloc,			0x06),1008	CMN_EVENT_HNP(awvalid_no_awready,		0x07),1009	CMN_EVENT_HNP(awready_no_awvalid,		0x08),1010	CMN_EVENT_HNP(wvalid_no_wready,			0x09),1011	CMN_EVENT_HNP(rrt_rd_occ_cnt_ovfl,		0x11),1012	CMN_EVENT_HNP(rdt_rd_occ_cnt_ovfl,		0x12),1013	CMN_EVENT_HNP(rrt_rd_alloc,			0x13),1014	CMN_EVENT_HNP(rdt_rd_alloc,			0x14),1015	CMN_EVENT_HNP(arvalid_no_arready,		0x15),1016	CMN_EVENT_HNP(arready_no_arvalid,		0x16),1017 1018	CMN_EVENT_XP(txflit_valid,			0x01),1019	CMN_EVENT_XP(txflit_stall,			0x02),1020	CMN_EVENT_XP_DAT(partial_dat_flit,		0x03),1021	/* We treat watchpoints as a special made-up class of XP events */1022	CMN_EVENT_ATTR(CMN_ANY, watchpoint_up, CMN_TYPE_WP, CMN_WP_UP),1023	CMN_EVENT_ATTR(CMN_ANY, watchpoint_down, CMN_TYPE_WP, CMN_WP_DOWN),1024 1025	CMN_EVENT_SBSX(CMN_ANY, rd_req,			0x01),1026	CMN_EVENT_SBSX(CMN_ANY, wr_req,			0x02),1027	CMN_EVENT_SBSX(CMN_ANY, cmo_req,		0x03),1028	CMN_EVENT_SBSX(CMN_ANY, txrsp_retryack,		0x04),1029	CMN_EVENT_SBSX(CMN_ANY, txdat_flitv,		0x05),1030	CMN_EVENT_SBSX(CMN_ANY, txrsp_flitv,		0x06),1031	CMN_EVENT_SBSX(CMN_ANY, rd_req_trkr_occ_cnt_ovfl, 0x11),1032	CMN_EVENT_SBSX(CMN_ANY, wr_req_trkr_occ_cnt_ovfl, 0x12),1033	CMN_EVENT_SBSX(CMN_ANY, cmo_req_trkr_occ_cnt_ovfl, 0x13),1034	CMN_EVENT_SBSX(CMN_ANY, wdb_occ_cnt_ovfl,	0x14),1035	CMN_EVENT_SBSX(CMN_ANY, rd_axi_trkr_occ_cnt_ovfl, 0x15),1036	CMN_EVENT_SBSX(CMN_ANY, cmo_axi_trkr_occ_cnt_ovfl, 0x16),1037	CMN_EVENT_SBSX(NOT_CMN600, rdb_occ_cnt_ovfl,	0x17),1038	CMN_EVENT_SBSX(CMN_ANY, arvalid_no_arready,	0x21),1039	CMN_EVENT_SBSX(CMN_ANY, awvalid_no_awready,	0x22),1040	CMN_EVENT_SBSX(CMN_ANY, wvalid_no_wready,	0x23),1041	CMN_EVENT_SBSX(CMN_ANY, txdat_stall,		0x24),1042	CMN_EVENT_SBSX(CMN_ANY, txrsp_stall,		0x25),1043 1044	CMN_EVENT_RNID(CMN_ANY, s0_rdata_beats,		0x01),1045	CMN_EVENT_RNID(CMN_ANY, s1_rdata_beats,		0x02),1046	CMN_EVENT_RNID(CMN_ANY, s2_rdata_beats,		0x03),1047	CMN_EVENT_RNID(CMN_ANY, rxdat_flits,		0x04),1048	CMN_EVENT_RNID(CMN_ANY, txdat_flits,		0x05),1049	CMN_EVENT_RNID(CMN_ANY, txreq_flits_total,	0x06),1050	CMN_EVENT_RNID(CMN_ANY, txreq_flits_retried,	0x07),1051	CMN_EVENT_RNID(CMN_ANY, rrt_occ_ovfl,		0x08),1052	CMN_EVENT_RNID(CMN_ANY, wrt_occ_ovfl,		0x09),1053	CMN_EVENT_RNID(CMN_ANY, txreq_flits_replayed,	0x0a),1054	CMN_EVENT_RNID(CMN_ANY, wrcancel_sent,		0x0b),1055	CMN_EVENT_RNID(CMN_ANY, s0_wdata_beats,		0x0c),1056	CMN_EVENT_RNID(CMN_ANY, s1_wdata_beats,		0x0d),1057	CMN_EVENT_RNID(CMN_ANY, s2_wdata_beats,		0x0e),1058	CMN_EVENT_RNID(CMN_ANY, rrt_alloc,		0x0f),1059	CMN_EVENT_RNID(CMN_ANY, wrt_alloc,		0x10),1060	CMN_EVENT_RNID(CMN600, rdb_unord,		0x11),1061	CMN_EVENT_RNID(CMN600, rdb_replay,		0x12),1062	CMN_EVENT_RNID(CMN600, rdb_hybrid,		0x13),1063	CMN_EVENT_RNID(CMN600, rdb_ord,			0x14),1064	CMN_EVENT_RNID(NOT_CMN600, padb_occ_ovfl,	0x11),1065	CMN_EVENT_RNID(NOT_CMN600, rpdb_occ_ovfl,	0x12),1066	CMN_EVENT_RNID(NOT_CMN600, rrt_occup_ovfl_slice1, 0x13),1067	CMN_EVENT_RNID(NOT_CMN600, rrt_occup_ovfl_slice2, 0x14),1068	CMN_EVENT_RNID(NOT_CMN600, rrt_occup_ovfl_slice3, 0x15),1069	CMN_EVENT_RNID(NOT_CMN600, wrt_throttled,	0x16),1070	CMN_EVENT_RNID(CMN700, ldb_full,		0x17),1071	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice0, 0x18),1072	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice1, 0x19),1073	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice2, 0x1a),1074	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice3, 0x1b),1075	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice0, 0x1c),1076	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice1, 0x1d),1077	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice2, 0x1e),1078	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice3, 0x1f),1079	CMN_EVENT_RNID(CMN700, rrt_burst_alloc,		0x20),1080	CMN_EVENT_RNID(CMN700, awid_hash,		0x21),1081	CMN_EVENT_RNID(CMN700, atomic_alloc,		0x22),1082	CMN_EVENT_RNID(CMN700, atomic_occ_ovfl,		0x23),1083 1084	CMN_EVENT_MTSX(tc_lookup,			0x01),1085	CMN_EVENT_MTSX(tc_fill,				0x02),1086	CMN_EVENT_MTSX(tc_miss,				0x03),1087	CMN_EVENT_MTSX(tdb_forward,			0x04),1088	CMN_EVENT_MTSX(tcq_hazard,			0x05),1089	CMN_EVENT_MTSX(tcq_rd_alloc,			0x06),1090	CMN_EVENT_MTSX(tcq_wr_alloc,			0x07),1091	CMN_EVENT_MTSX(tcq_cmo_alloc,			0x08),1092	CMN_EVENT_MTSX(axi_rd_req,			0x09),1093	CMN_EVENT_MTSX(axi_wr_req,			0x0a),1094	CMN_EVENT_MTSX(tcq_occ_cnt_ovfl,		0x0b),1095	CMN_EVENT_MTSX(tdb_occ_cnt_ovfl,		0x0c),1096 1097	CMN_EVENT_CXRA(CMN_ANY, rht_occ,		0x01),1098	CMN_EVENT_CXRA(CMN_ANY, sht_occ,		0x02),1099	CMN_EVENT_CXRA(CMN_ANY, rdb_occ,		0x03),1100	CMN_EVENT_CXRA(CMN_ANY, wdb_occ,		0x04),1101	CMN_EVENT_CXRA(CMN_ANY, ssb_occ,		0x05),1102	CMN_EVENT_CXRA(CMN_ANY, snp_bcasts,		0x06),1103	CMN_EVENT_CXRA(CMN_ANY, req_chains,		0x07),1104	CMN_EVENT_CXRA(CMN_ANY, req_chain_avglen,	0x08),1105	CMN_EVENT_CXRA(CMN_ANY, chirsp_stalls,		0x09),1106	CMN_EVENT_CXRA(CMN_ANY, chidat_stalls,		0x0a),1107	CMN_EVENT_CXRA(CMN_ANY, cxreq_pcrd_stalls_link0, 0x0b),1108	CMN_EVENT_CXRA(CMN_ANY, cxreq_pcrd_stalls_link1, 0x0c),1109	CMN_EVENT_CXRA(CMN_ANY, cxreq_pcrd_stalls_link2, 0x0d),1110	CMN_EVENT_CXRA(CMN_ANY, cxdat_pcrd_stalls_link0, 0x0e),1111	CMN_EVENT_CXRA(CMN_ANY, cxdat_pcrd_stalls_link1, 0x0f),1112	CMN_EVENT_CXRA(CMN_ANY, cxdat_pcrd_stalls_link2, 0x10),1113	CMN_EVENT_CXRA(CMN_ANY, external_chirsp_stalls,	0x11),1114	CMN_EVENT_CXRA(CMN_ANY, external_chidat_stalls,	0x12),1115	CMN_EVENT_CXRA(NOT_CMN600, cxmisc_pcrd_stalls_link0, 0x13),1116	CMN_EVENT_CXRA(NOT_CMN600, cxmisc_pcrd_stalls_link1, 0x14),1117	CMN_EVENT_CXRA(NOT_CMN600, cxmisc_pcrd_stalls_link2, 0x15),1118 1119	CMN_EVENT_CXHA(rddatbyp,			0x21),1120	CMN_EVENT_CXHA(chirsp_up_stall,			0x22),1121	CMN_EVENT_CXHA(chidat_up_stall,			0x23),1122	CMN_EVENT_CXHA(snppcrd_link0_stall,		0x24),1123	CMN_EVENT_CXHA(snppcrd_link1_stall,		0x25),1124	CMN_EVENT_CXHA(snppcrd_link2_stall,		0x26),1125	CMN_EVENT_CXHA(reqtrk_occ,			0x27),1126	CMN_EVENT_CXHA(rdb_occ,				0x28),1127	CMN_EVENT_CXHA(rdbyp_occ,			0x29),1128	CMN_EVENT_CXHA(wdb_occ,				0x2a),1129	CMN_EVENT_CXHA(snptrk_occ,			0x2b),1130	CMN_EVENT_CXHA(sdb_occ,				0x2c),1131	CMN_EVENT_CXHA(snphaz_occ,			0x2d),1132 1133	CMN_EVENT_CCRA(rht_occ,				0x41),1134	CMN_EVENT_CCRA(sht_occ,				0x42),1135	CMN_EVENT_CCRA(rdb_occ,				0x43),1136	CMN_EVENT_CCRA(wdb_occ,				0x44),1137	CMN_EVENT_CCRA(ssb_occ,				0x45),1138	CMN_EVENT_CCRA(snp_bcasts,			0x46),1139	CMN_EVENT_CCRA(req_chains,			0x47),1140	CMN_EVENT_CCRA(req_chain_avglen,		0x48),1141	CMN_EVENT_CCRA(chirsp_stalls,			0x49),1142	CMN_EVENT_CCRA(chidat_stalls,			0x4a),1143	CMN_EVENT_CCRA(cxreq_pcrd_stalls_link0,		0x4b),1144	CMN_EVENT_CCRA(cxreq_pcrd_stalls_link1,		0x4c),1145	CMN_EVENT_CCRA(cxreq_pcrd_stalls_link2,		0x4d),1146	CMN_EVENT_CCRA(cxdat_pcrd_stalls_link0,		0x4e),1147	CMN_EVENT_CCRA(cxdat_pcrd_stalls_link1,		0x4f),1148	CMN_EVENT_CCRA(cxdat_pcrd_stalls_link2,		0x50),1149	CMN_EVENT_CCRA(external_chirsp_stalls,		0x51),1150	CMN_EVENT_CCRA(external_chidat_stalls,		0x52),1151	CMN_EVENT_CCRA(cxmisc_pcrd_stalls_link0,	0x53),1152	CMN_EVENT_CCRA(cxmisc_pcrd_stalls_link1,	0x54),1153	CMN_EVENT_CCRA(cxmisc_pcrd_stalls_link2,	0x55),1154	CMN_EVENT_CCRA(rht_alloc,			0x56),1155	CMN_EVENT_CCRA(sht_alloc,			0x57),1156	CMN_EVENT_CCRA(rdb_alloc,			0x58),1157	CMN_EVENT_CCRA(wdb_alloc,			0x59),1158	CMN_EVENT_CCRA(ssb_alloc,			0x5a),1159 1160	CMN_EVENT_CCHA(CMN_ANY, rddatbyp,		0x61),1161	CMN_EVENT_CCHA(CMN_ANY, chirsp_up_stall,	0x62),1162	CMN_EVENT_CCHA(CMN_ANY, chidat_up_stall,	0x63),1163	CMN_EVENT_CCHA(CMN_ANY, snppcrd_link0_stall,	0x64),1164	CMN_EVENT_CCHA(CMN_ANY, snppcrd_link1_stall,	0x65),1165	CMN_EVENT_CCHA(CMN_ANY, snppcrd_link2_stall,	0x66),1166	CMN_EVENT_CCHA(CMN_ANY, reqtrk_occ,		0x67),1167	CMN_EVENT_CCHA(CMN_ANY, rdb_occ,		0x68),1168	CMN_EVENT_CCHA(CMN_ANY, rdbyp_occ,		0x69),1169	CMN_EVENT_CCHA(CMN_ANY, wdb_occ,		0x6a),1170	CMN_EVENT_CCHA(CMN_ANY, snptrk_occ,		0x6b),1171	CMN_EVENT_CCHA(CMN_ANY, sdb_occ,		0x6c),1172	CMN_EVENT_CCHA(CMN_ANY, snphaz_occ,		0x6d),1173	CMN_EVENT_CCHA(CMN_ANY, reqtrk_alloc,		0x6e),1174	CMN_EVENT_CCHA(CMN_ANY, rdb_alloc,		0x6f),1175	CMN_EVENT_CCHA(CMN_ANY, rdbyp_alloc,		0x70),1176	CMN_EVENT_CCHA(CMN_ANY, wdb_alloc,		0x71),1177	CMN_EVENT_CCHA(CMN_ANY, snptrk_alloc,		0x72),1178	CMN_EVENT_CCHA(CMN_ANY, db_alloc,		0x73),1179	CMN_EVENT_CCHA(CMN_ANY, snphaz_alloc,		0x74),1180	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_req_occ,		0x75),1181	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_req_alloc,	0x76),1182	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_req_occ,	0x77),1183	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_req_alloc,	0x78),1184	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_req_occ,	0x79),1185	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_req_alloc,	0x7a),1186	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_reg_req_occ,	0x7b),1187	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_reg_req_alloc,	0x7c),1188	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_rsvd_req_occ,	0x7d),1189	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_rsvd_req_alloc,	0x7e),1190	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_dat_occ,		0x7f),1191	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_dat_alloc,	0x80),1192	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_dat_occ,	0x81),1193	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_dat_alloc,	0x82),1194	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_dat_occ,	0x83),1195	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_dat_alloc,	0x84),1196	CMN_EVENT_CCHA(CMNS3, chirsp1_up_stall,		0x85),1197 1198	CMN_EVENT_CCLA(rx_cxs,				0x21),1199	CMN_EVENT_CCLA(tx_cxs,				0x22),1200	CMN_EVENT_CCLA(rx_cxs_avg_size,			0x23),1201	CMN_EVENT_CCLA(tx_cxs_avg_size,			0x24),1202	CMN_EVENT_CCLA(tx_cxs_lcrd_backpressure,	0x25),1203	CMN_EVENT_CCLA(link_crdbuf_occ,			0x26),1204	CMN_EVENT_CCLA(link_crdbuf_alloc,		0x27),1205	CMN_EVENT_CCLA(pfwd_rcvr_cxs,			0x28),1206	CMN_EVENT_CCLA(pfwd_sndr_num_flits,		0x29),1207	CMN_EVENT_CCLA(pfwd_sndr_stalls_static_crd,	0x2a),1208	CMN_EVENT_CCLA(pfwd_sndr_stalls_dynmaic_crd,	0x2b),1209 1210	CMN_EVENT_HNS_HBT(cache_miss,			0x01),1211	CMN_EVENT_HNS_HBT(slc_sf_cache_access,		0x02),1212	CMN_EVENT_HNS_HBT(cache_fill,			0x03),1213	CMN_EVENT_HNS_HBT(pocq_retry,			0x04),1214	CMN_EVENT_HNS_HBT(pocq_reqs_recvd,		0x05),1215	CMN_EVENT_HNS_HBT(sf_hit,			0x06),1216	CMN_EVENT_HNS_HBT(sf_evictions,			0x07),1217	CMN_EVENT_HNS(dir_snoops_sent,			0x08),1218	CMN_EVENT_HNS(brd_snoops_sent,			0x09),1219	CMN_EVENT_HNS_HBT(slc_eviction,			0x0a),1220	CMN_EVENT_HNS_HBT(slc_fill_invalid_way,		0x0b),1221	CMN_EVENT_HNS(mc_retries_local,			0x0c),1222	CMN_EVENT_HNS_SNH(mc_reqs_local,		0x0d),1223	CMN_EVENT_HNS(qos_hh_retry,			0x0e),1224	CMN_EVENT_HNS_OCC(qos_pocq_occupancy,		0x0f),1225	CMN_EVENT_HNS(pocq_addrhaz,			0x10),1226	CMN_EVENT_HNS(pocq_atomic_addrhaz,		0x11),1227	CMN_EVENT_HNS(ld_st_swp_adq_full,		0x12),1228	CMN_EVENT_HNS(cmp_adq_full,			0x13),1229	CMN_EVENT_HNS(txdat_stall,			0x14),1230	CMN_EVENT_HNS(txrsp_stall,			0x15),1231	CMN_EVENT_HNS(seq_full,				0x16),1232	CMN_EVENT_HNS(seq_hit,				0x17),1233	CMN_EVENT_HNS(snp_sent,				0x18),1234	CMN_EVENT_HNS(sfbi_dir_snp_sent,		0x19),1235	CMN_EVENT_HNS(sfbi_brd_snp_sent,		0x1a),1236	CMN_EVENT_HNS(intv_dirty,			0x1c),1237	CMN_EVENT_HNS(stash_snp_sent,			0x1d),1238	CMN_EVENT_HNS(stash_data_pull,			0x1e),1239	CMN_EVENT_HNS(snp_fwded,			0x1f),1240	CMN_EVENT_HNS(atomic_fwd,			0x20),1241	CMN_EVENT_HNS(mpam_hardlim,			0x21),1242	CMN_EVENT_HNS(mpam_softlim,			0x22),1243	CMN_EVENT_HNS(snp_sent_cluster,			0x23),1244	CMN_EVENT_HNS(sf_imprecise_evict,		0x24),1245	CMN_EVENT_HNS(sf_evict_shared_line,		0x25),1246	CMN_EVENT_HNS_CLS(pocq_class_occup,		0x26),1247	CMN_EVENT_HNS_CLS(pocq_class_retry,		0x27),1248	CMN_EVENT_HNS_CLS(class_mc_reqs_local,		0x28),1249	CMN_EVENT_HNS_CLS(class_cgnt_cmin,		0x29),1250	CMN_EVENT_HNS_SNT(sn_throttle,			0x2a),1251	CMN_EVENT_HNS_SNT(sn_throttle_min,		0x2b),1252	CMN_EVENT_HNS(sf_precise_to_imprecise,		0x2c),1253	CMN_EVENT_HNS(snp_intv_cln,			0x2d),1254	CMN_EVENT_HNS(nc_excl,				0x2e),1255	CMN_EVENT_HNS(excl_mon_ovfl,			0x2f),1256	CMN_EVENT_HNS(snp_req_recvd,			0x30),1257	CMN_EVENT_HNS(snp_req_byp_pocq,			0x31),1258	CMN_EVENT_HNS(dir_ccgha_snp_sent,		0x32),1259	CMN_EVENT_HNS(brd_ccgha_snp_sent,		0x33),1260	CMN_EVENT_HNS(ccgha_snp_stall,			0x34),1261	CMN_EVENT_HNS(lbt_req_hardlim,			0x35),1262	CMN_EVENT_HNS(hbt_req_hardlim,			0x36),1263	CMN_EVENT_HNS(sf_reupdate,			0x37),1264	CMN_EVENT_HNS(excl_sf_imprecise,		0x38),1265	CMN_EVENT_HNS(snp_pocq_addrhaz,			0x39),1266	CMN_EVENT_HNS(mc_retries_remote,		0x3a),1267	CMN_EVENT_HNS_SNH(mc_reqs_remote,		0x3b),1268	CMN_EVENT_HNS_CLS(class_mc_reqs_remote,		0x3c),1269 1270	NULL1271};1272 1273static const struct attribute_group arm_cmn_event_attrs_group = {1274	.name = "events",1275	.attrs = arm_cmn_event_attrs,1276	.is_visible = arm_cmn_event_attr_is_visible,1277};1278 1279static ssize_t arm_cmn_format_show(struct device *dev,1280				   struct device_attribute *attr, char *buf)1281{1282	struct arm_cmn_format_attr *fmt = container_of(attr, typeof(*fmt), attr);1283 1284	if (!fmt->config)1285		return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field);1286 1287	return sysfs_emit(buf, "config%d:%*pbl\n", fmt->config, 64, &fmt->field);1288}1289 1290#define _CMN_FORMAT_ATTR(_name, _cfg, _fld)				\1291	(&((struct arm_cmn_format_attr[]) {{				\1292		.attr = __ATTR(_name, 0444, arm_cmn_format_show, NULL),	\1293		.config = _cfg,						\1294		.field = _fld,						\1295	}})[0].attr.attr)1296#define CMN_FORMAT_ATTR(_name, _fld)	_CMN_FORMAT_ATTR(_name, 0, _fld)1297 1298static struct attribute *arm_cmn_format_attrs[] = {1299	CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE),1300	CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID),1301	CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID),1302	CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID),1303	CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID),1304 1305	CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL),1306	CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL),1307	CMN_FORMAT_ATTR(wp_grp, CMN_CONFIG_WP_GRP),1308	CMN_FORMAT_ATTR(wp_exclusive, CMN_CONFIG_WP_EXCLUSIVE),1309	CMN_FORMAT_ATTR(wp_combine, CMN_CONFIG_WP_COMBINE),1310 1311	_CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL),1312	_CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK),1313 1314	NULL1315};1316 1317static const struct attribute_group arm_cmn_format_attrs_group = {1318	.name = "format",1319	.attrs = arm_cmn_format_attrs,1320};1321 1322static ssize_t arm_cmn_cpumask_show(struct device *dev,1323				    struct device_attribute *attr, char *buf)1324{1325	struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));1326 1327	return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu));1328}1329 1330static struct device_attribute arm_cmn_cpumask_attr =1331		__ATTR(cpumask, 0444, arm_cmn_cpumask_show, NULL);1332 1333static ssize_t arm_cmn_identifier_show(struct device *dev,1334				       struct device_attribute *attr, char *buf)1335{1336	struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));1337 1338	return sysfs_emit(buf, "%03x%02x\n", cmn->part, cmn->rev);1339}1340 1341static struct device_attribute arm_cmn_identifier_attr =1342		__ATTR(identifier, 0444, arm_cmn_identifier_show, NULL);1343 1344static struct attribute *arm_cmn_other_attrs[] = {1345	&arm_cmn_cpumask_attr.attr,1346	&arm_cmn_identifier_attr.attr,1347	NULL,1348};1349 1350static const struct attribute_group arm_cmn_other_attrs_group = {1351	.attrs = arm_cmn_other_attrs,1352};1353 1354static const struct attribute_group *arm_cmn_attr_groups[] = {1355	&arm_cmn_event_attrs_group,1356	&arm_cmn_format_attrs_group,1357	&arm_cmn_other_attrs_group,1358	NULL1359};1360 1361static int arm_cmn_find_free_wp_idx(struct arm_cmn_dtm *dtm,1362				    struct perf_event *event)1363{1364	int wp_idx = CMN_EVENT_EVENTID(event);1365 1366	if (dtm->wp_event[wp_idx] >= 0)1367		if (dtm->wp_event[++wp_idx] >= 0)1368			return -ENOSPC;1369 1370	return wp_idx;1371}1372 1373static int arm_cmn_get_assigned_wp_idx(struct perf_event *event,1374				       struct arm_cmn_hw_event *hw,1375				       unsigned int pos)1376{1377	return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw->wp_idx, pos);1378}1379 1380static void arm_cmn_claim_wp_idx(struct arm_cmn_dtm *dtm,1381				 struct perf_event *event,1382				 unsigned int dtc, int wp_idx,1383				 unsigned int pos)1384{1385	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1386 1387	dtm->wp_event[wp_idx] = hw->dtc_idx[dtc];1388	arm_cmn_set_wp_idx(hw->wp_idx, pos, wp_idx - CMN_EVENT_EVENTID(event));1389}1390 1391static u32 arm_cmn_wp_config(struct perf_event *event, int wp_idx)1392{1393	u32 config;1394	u32 dev = CMN_EVENT_WP_DEV_SEL(event);1395	u32 chn = CMN_EVENT_WP_CHN_SEL(event);1396	u32 grp = CMN_EVENT_WP_GRP(event);1397	u32 exc = CMN_EVENT_WP_EXCLUSIVE(event);1398	u32 combine = CMN_EVENT_WP_COMBINE(event);1399	bool is_cmn600 = to_cmn(event->pmu)->part == PART_CMN600;1400 1401	/* CMN-600 supports only primary and secondary matching groups */1402	if (is_cmn600)1403		grp &= 1;1404 1405	config = FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL, dev) |1406		 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) |1407		 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp) |1408		 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL2, dev >> 1);1409	if (exc)1410		config |= is_cmn600 ? CMN600_WPn_CONFIG_WP_EXCLUSIVE :1411				      CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE;1412 1413	/*  wp_combine is available only on WP0 and WP2 */1414	if (combine && !(wp_idx & 0x1))1415		config |= is_cmn600 ? CMN600_WPn_CONFIG_WP_COMBINE :1416				      CMN_DTM_WPn_CONFIG_WP_COMBINE;1417	return config;1418}1419 1420static void arm_cmn_set_state(struct arm_cmn *cmn, u32 state)1421{1422	if (!cmn->state)1423		writel_relaxed(0, CMN_DT_PMCR(&cmn->dtc[0]));1424	cmn->state |= state;1425}1426 1427static void arm_cmn_clear_state(struct arm_cmn *cmn, u32 state)1428{1429	cmn->state &= ~state;1430	if (!cmn->state)1431		writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN,1432			       CMN_DT_PMCR(&cmn->dtc[0]));1433}1434 1435static void arm_cmn_pmu_enable(struct pmu *pmu)1436{1437	arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_DISABLED);1438}1439 1440static void arm_cmn_pmu_disable(struct pmu *pmu)1441{1442	arm_cmn_set_state(to_cmn(pmu), CMN_STATE_DISABLED);1443}1444 1445static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw,1446			    bool snapshot)1447{1448	struct arm_cmn_dtm *dtm = NULL;1449	struct arm_cmn_node *dn;1450	unsigned int i, offset, dtm_idx;1451	u64 reg, count = 0;1452 1453	offset = snapshot ? CMN_DTM_PMEVCNTSR : CMN_DTM_PMEVCNT;1454	for_each_hw_dn(hw, dn, i) {1455		if (dtm != &cmn->dtms[dn->dtm]) {1456			dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset;1457			reg = readq_relaxed(dtm->base + offset);1458		}1459		dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);1460		count += (u16)(reg >> (dtm_idx * 16));1461	}1462	return count;1463}1464 1465static u64 arm_cmn_read_cc(struct arm_cmn_dtc *dtc)1466{1467	void __iomem *pmccntr = CMN_DT_PMCCNTR(dtc);1468	u64 val = readq_relaxed(pmccntr);1469 1470	writeq_relaxed(CMN_CC_INIT, pmccntr);1471	return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1);1472}1473 1474static u32 arm_cmn_read_counter(struct arm_cmn_dtc *dtc, int idx)1475{1476	void __iomem *pmevcnt = CMN_DT_PMEVCNT(dtc, idx);1477	u32 val = readl_relaxed(pmevcnt);1478 1479	writel_relaxed(CMN_COUNTER_INIT, pmevcnt);1480	return val - CMN_COUNTER_INIT;1481}1482 1483static void arm_cmn_init_counter(struct perf_event *event)1484{1485	struct arm_cmn *cmn = to_cmn(event->pmu);1486	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1487	u64 count;1488 1489	for_each_hw_dtc_idx(hw, i, idx) {1490		writel_relaxed(CMN_COUNTER_INIT, CMN_DT_PMEVCNT(&cmn->dtc[i], idx));1491		cmn->dtc[i].counters[idx] = event;1492	}1493 1494	count = arm_cmn_read_dtm(cmn, hw, false);1495	local64_set(&event->hw.prev_count, count);1496}1497 1498static void arm_cmn_event_read(struct perf_event *event)1499{1500	struct arm_cmn *cmn = to_cmn(event->pmu);1501	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1502	u64 delta, new, prev;1503	unsigned long flags;1504 1505	if (CMN_EVENT_TYPE(event) == CMN_TYPE_DTC) {1506		delta = arm_cmn_read_cc(cmn->dtc + hw->dtc_idx[0]);1507		local64_add(delta, &event->count);1508		return;1509	}1510	new = arm_cmn_read_dtm(cmn, hw, false);1511	prev = local64_xchg(&event->hw.prev_count, new);1512 1513	delta = new - prev;1514 1515	local_irq_save(flags);1516	for_each_hw_dtc_idx(hw, i, idx) {1517		new = arm_cmn_read_counter(cmn->dtc + i, idx);1518		delta += new << 16;1519	}1520	local_irq_restore(flags);1521	local64_add(delta, &event->count);1522}1523 1524static int arm_cmn_set_event_sel_hi(struct arm_cmn_node *dn,1525				    enum cmn_filter_select fsel, u8 occupid)1526{1527	u64 reg;1528 1529	if (fsel == SEL_NONE)1530		return 0;1531 1532	if (!dn->occupid[fsel].count) {1533		dn->occupid[fsel].val = occupid;1534		reg = FIELD_PREP(CMN__PMU_CBUSY_SNTHROTTLE_SEL,1535				 dn->occupid[SEL_CBUSY_SNTHROTTLE_SEL].val) |1536		      FIELD_PREP(CMN__PMU_SN_HOME_SEL,1537				 dn->occupid[SEL_SN_HOME_SEL].val) |1538		      FIELD_PREP(CMN__PMU_HBT_LBT_SEL,1539				 dn->occupid[SEL_HBT_LBT_SEL].val) |1540		      FIELD_PREP(CMN__PMU_CLASS_OCCUP_ID,1541				 dn->occupid[SEL_CLASS_OCCUP_ID].val) |1542		      FIELD_PREP(CMN__PMU_OCCUP1_ID,1543				 dn->occupid[SEL_OCCUP1ID].val);1544		writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4);1545	} else if (dn->occupid[fsel].val != occupid) {1546		return -EBUSY;1547	}1548	dn->occupid[fsel].count++;1549	return 0;1550}1551 1552static void arm_cmn_set_event_sel_lo(struct arm_cmn_node *dn, int dtm_idx,1553				     int eventid, bool wide_sel)1554{1555	if (wide_sel) {1556		dn->event_w[dtm_idx] = eventid;1557		writeq_relaxed(le64_to_cpu(dn->event_sel_w), dn->pmu_base + CMN_PMU_EVENT_SEL);1558	} else {1559		dn->event[dtm_idx] = eventid;1560		writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL);1561	}1562}1563 1564static void arm_cmn_event_start(struct perf_event *event, int flags)1565{1566	struct arm_cmn *cmn = to_cmn(event->pmu);1567	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1568	struct arm_cmn_node *dn;1569	enum cmn_node_type type = CMN_EVENT_TYPE(event);1570	int i;1571 1572	if (type == CMN_TYPE_DTC) {1573		struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0];1574 1575		writel_relaxed(CMN_DT_DTC_CTL_DT_EN | CMN_DT_DTC_CTL_CG_DISABLE,1576			       dtc->base + CMN_DT_DTC_CTL);1577		writeq_relaxed(CMN_CC_INIT, CMN_DT_PMCCNTR(dtc));1578		dtc->cc_active = true;1579	} else if (type == CMN_TYPE_WP) {1580		u64 val = CMN_EVENT_WP_VAL(event);1581		u64 mask = CMN_EVENT_WP_MASK(event);1582 1583		for_each_hw_dn(hw, dn, i) {1584			void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset);1585			int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i);1586 1587			writeq_relaxed(val, base + CMN_DTM_WPn_VAL(wp_idx));1588			writeq_relaxed(mask, base + CMN_DTM_WPn_MASK(wp_idx));1589		}1590	} else for_each_hw_dn(hw, dn, i) {1591		int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);1592 1593		arm_cmn_set_event_sel_lo(dn, dtm_idx, CMN_EVENT_EVENTID(event),1594					 hw->wide_sel);1595	}1596}1597 1598static void arm_cmn_event_stop(struct perf_event *event, int flags)1599{1600	struct arm_cmn *cmn = to_cmn(event->pmu);1601	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1602	struct arm_cmn_node *dn;1603	enum cmn_node_type type = CMN_EVENT_TYPE(event);1604	int i;1605 1606	if (type == CMN_TYPE_DTC) {1607		struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0];1608 1609		dtc->cc_active = false;1610		writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL);1611	} else if (type == CMN_TYPE_WP) {1612		for_each_hw_dn(hw, dn, i) {1613			void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset);1614			int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i);1615 1616			writeq_relaxed(0, base + CMN_DTM_WPn_MASK(wp_idx));1617			writeq_relaxed(~0ULL, base + CMN_DTM_WPn_VAL(wp_idx));1618		}1619	} else for_each_hw_dn(hw, dn, i) {1620		int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);1621 1622		arm_cmn_set_event_sel_lo(dn, dtm_idx, 0, hw->wide_sel);1623	}1624 1625	arm_cmn_event_read(event);1626}1627 1628struct arm_cmn_val {1629	u8 dtm_count[CMN_MAX_DTMS];1630	u8 occupid[CMN_MAX_DTMS][SEL_MAX];1631	u8 wp[CMN_MAX_DTMS][4];1632	u8 wp_combine[CMN_MAX_DTMS][2];1633	int dtc_count[CMN_MAX_DTCS];1634	bool cycles;1635};1636 1637static int arm_cmn_val_find_free_wp_config(struct perf_event *event,1638					  struct arm_cmn_val *val, int dtm)1639{1640	int wp_idx = CMN_EVENT_EVENTID(event);1641 1642	if (val->wp[dtm][wp_idx])1643		if (val->wp[dtm][++wp_idx])1644			return -ENOSPC;1645 1646	return wp_idx;1647}1648 1649static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val,1650				  struct perf_event *event)1651{1652	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1653	struct arm_cmn_node *dn;1654	enum cmn_node_type type;1655	int i;1656 1657	if (is_software_event(event))1658		return;1659 1660	type = CMN_EVENT_TYPE(event);1661	if (type == CMN_TYPE_DTC) {1662		val->cycles = true;1663		return;1664	}1665 1666	for_each_hw_dtc_idx(hw, dtc, idx)1667		val->dtc_count[dtc]++;1668 1669	for_each_hw_dn(hw, dn, i) {1670		int wp_idx, dtm = dn->dtm, sel = hw->filter_sel;1671 1672		val->dtm_count[dtm]++;1673 1674		if (sel > SEL_NONE)1675			val->occupid[dtm][sel] = CMN_EVENT_OCCUPID(event) + 1;1676 1677		if (type != CMN_TYPE_WP)1678			continue;1679 1680		wp_idx = arm_cmn_val_find_free_wp_config(event, val, dtm);1681		val->wp[dtm][wp_idx] = 1;1682		val->wp_combine[dtm][wp_idx >> 1] += !!CMN_EVENT_WP_COMBINE(event);1683	}1684}1685 1686static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event)1687{1688	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1689	struct arm_cmn_node *dn;1690	struct perf_event *sibling, *leader = event->group_leader;1691	enum cmn_node_type type;1692	struct arm_cmn_val *val;1693	int i, ret = -EINVAL;1694 1695	if (leader == event)1696		return 0;1697 1698	if (event->pmu != leader->pmu && !is_software_event(leader))1699		return -EINVAL;1700 1701	val = kzalloc(sizeof(*val), GFP_KERNEL);1702	if (!val)1703		return -ENOMEM;1704 1705	arm_cmn_val_add_event(cmn, val, leader);1706 1707	for_each_sibling_event(sibling, leader)1708		arm_cmn_val_add_event(cmn, val, sibling);1709 1710	type = CMN_EVENT_TYPE(event);1711	if (type == CMN_TYPE_DTC) {1712		ret = val->cycles ? -EINVAL : 0;1713		goto done;1714	}1715 1716	for (i = 0; i < CMN_MAX_DTCS; i++)1717		if (val->dtc_count[i] == CMN_DT_NUM_COUNTERS)1718			goto done;1719 1720	for_each_hw_dn(hw, dn, i) {1721		int wp_idx, dtm = dn->dtm, sel = hw->filter_sel;1722 1723		if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS)1724			goto done;1725 1726		if (sel > SEL_NONE && val->occupid[dtm][sel] &&1727		    val->occupid[dtm][sel] != CMN_EVENT_OCCUPID(event) + 1)1728			goto done;1729 1730		if (type != CMN_TYPE_WP)1731			continue;1732 1733		wp_idx = arm_cmn_val_find_free_wp_config(event, val, dtm);1734		if (wp_idx < 0)1735			goto done;1736 1737		if (wp_idx & 1 &&1738		    val->wp_combine[dtm][wp_idx >> 1] != !!CMN_EVENT_WP_COMBINE(event))1739			goto done;1740	}1741 1742	ret = 0;1743done:1744	kfree(val);1745	return ret;1746}1747 1748static enum cmn_filter_select arm_cmn_filter_sel(const struct arm_cmn *cmn,1749						 enum cmn_node_type type,1750						 unsigned int eventid)1751{1752	struct arm_cmn_event_attr *e;1753	enum cmn_model model = arm_cmn_model(cmn);1754 1755	for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) {1756		e = container_of(arm_cmn_event_attrs[i], typeof(*e), attr.attr);1757		if (e->model & model && e->type == type && e->eventid == eventid)1758			return e->fsel;1759	}1760	return SEL_NONE;1761}1762 1763 1764static int arm_cmn_event_init(struct perf_event *event)1765{1766	struct arm_cmn *cmn = to_cmn(event->pmu);1767	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1768	struct arm_cmn_node *dn;1769	enum cmn_node_type type;1770	bool bynodeid;1771	u16 nodeid, eventid;1772 1773	if (event->attr.type != event->pmu->type)1774		return -ENOENT;1775 1776	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)1777		return -EINVAL;1778 1779	event->cpu = cmn->cpu;1780	if (event->cpu < 0)1781		return -EINVAL;1782 1783	type = CMN_EVENT_TYPE(event);1784	/* DTC events (i.e. cycles) already have everything they need */1785	if (type == CMN_TYPE_DTC)1786		return arm_cmn_validate_group(cmn, event);1787 1788	eventid = CMN_EVENT_EVENTID(event);1789	/* For watchpoints we need the actual XP node here */1790	if (type == CMN_TYPE_WP) {1791		type = CMN_TYPE_XP;1792		/* ...and we need a "real" direction */1793		if (eventid != CMN_WP_UP && eventid != CMN_WP_DOWN)1794			return -EINVAL;1795		/* ...but the DTM may depend on which port we're watching */1796		if (cmn->multi_dtm)1797			hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2;1798	} else if (type == CMN_TYPE_XP &&1799		   (cmn->part == PART_CMN700 || cmn->part == PART_CMN_S3)) {1800		hw->wide_sel = true;1801	}1802 1803	/* This is sufficiently annoying to recalculate, so cache it */1804	hw->filter_sel = arm_cmn_filter_sel(cmn, type, eventid);1805 1806	bynodeid = CMN_EVENT_BYNODEID(event);1807	nodeid = CMN_EVENT_NODEID(event);1808 1809	hw->dn = arm_cmn_node(cmn, type);1810	if (!hw->dn)1811		return -EINVAL;1812 1813	memset(hw->dtc_idx, -1, sizeof(hw->dtc_idx));1814	for (dn = hw->dn; dn->type == type; dn++) {1815		if (bynodeid && dn->id != nodeid) {1816			hw->dn++;1817			continue;1818		}1819		hw->num_dns++;1820		if (dn->dtc < 0)1821			memset(hw->dtc_idx, 0, cmn->num_dtcs);1822		else1823			hw->dtc_idx[dn->dtc] = 0;1824 1825		if (bynodeid)1826			break;1827	}1828 1829	if (!hw->num_dns) {1830		dev_dbg(cmn->dev, "invalid node 0x%x type 0x%x\n", nodeid, type);1831		return -EINVAL;1832	}1833 1834	return arm_cmn_validate_group(cmn, event);1835}1836 1837static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event,1838				int i)1839{1840	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1841	enum cmn_node_type type = CMN_EVENT_TYPE(event);1842 1843	while (i--) {1844		struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset;1845		unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);1846 1847		if (type == CMN_TYPE_WP) {1848			int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i);1849 1850			dtm->wp_event[wp_idx] = -1;1851		}1852 1853		if (hw->filter_sel > SEL_NONE)1854			hw->dn[i].occupid[hw->filter_sel].count--;1855 1856		dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx);1857		writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG);1858	}1859	memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx));1860	memset(hw->wp_idx, 0, sizeof(hw->wp_idx));1861 1862	for_each_hw_dtc_idx(hw, j, idx)1863		cmn->dtc[j].counters[idx] = NULL;1864}1865 1866static int arm_cmn_event_add(struct perf_event *event, int flags)1867{1868	struct arm_cmn *cmn = to_cmn(event->pmu);1869	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1870	struct arm_cmn_node *dn;1871	enum cmn_node_type type = CMN_EVENT_TYPE(event);1872	unsigned int input_sel, i = 0;1873 1874	if (type == CMN_TYPE_DTC) {1875		while (cmn->dtc[i].cycles)1876			if (++i == cmn->num_dtcs)1877				return -ENOSPC;1878 1879		cmn->dtc[i].cycles = event;1880		hw->dtc_idx[0] = i;1881 1882		if (flags & PERF_EF_START)1883			arm_cmn_event_start(event, 0);1884		return 0;1885	}1886 1887	/* Grab the global counters first... */1888	for_each_hw_dtc_idx(hw, j, idx) {1889		if (cmn->part == PART_CMN600 && j > 0) {1890			idx = hw->dtc_idx[0];1891		} else {1892			idx = 0;1893			while (cmn->dtc[j].counters[idx])1894				if (++idx == CMN_DT_NUM_COUNTERS)1895					return -ENOSPC;1896		}1897		hw->dtc_idx[j] = idx;1898	}1899 1900	/* ...then the local counters to feed them */1901	for_each_hw_dn(hw, dn, i) {1902		struct arm_cmn_dtm *dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset;1903		unsigned int dtm_idx, shift, d = max_t(int, dn->dtc, 0);1904		u64 reg;1905 1906		dtm_idx = 0;1907		while (dtm->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx))1908			if (++dtm_idx == CMN_DTM_NUM_COUNTERS)1909				goto free_dtms;1910 1911		if (type == CMN_TYPE_XP) {1912			input_sel = CMN__PMEVCNT0_INPUT_SEL_XP + dtm_idx;1913		} else if (type == CMN_TYPE_WP) {1914			int tmp, wp_idx;1915			u32 cfg;1916 1917			wp_idx = arm_cmn_find_free_wp_idx(dtm, event);1918			if (wp_idx < 0)1919				goto free_dtms;1920 1921			cfg = arm_cmn_wp_config(event, wp_idx);1922 1923			tmp = dtm->wp_event[wp_idx ^ 1];1924			if (tmp >= 0 && CMN_EVENT_WP_COMBINE(event) !=1925					CMN_EVENT_WP_COMBINE(cmn->dtc[d].counters[tmp]))1926				goto free_dtms;1927 1928			input_sel = CMN__PMEVCNT0_INPUT_SEL_WP + wp_idx;1929 1930			arm_cmn_claim_wp_idx(dtm, event, d, wp_idx, i);1931			writel_relaxed(cfg, dtm->base + CMN_DTM_WPn_CONFIG(wp_idx));1932		} else {1933			struct arm_cmn_nodeid nid = arm_cmn_nid(dn);1934 1935			if (cmn->multi_dtm)1936				nid.port %= 2;1937 1938			input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx +1939				    (nid.port << 4) + (nid.dev << 2);1940 1941			if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_OCCUPID(event)))1942				goto free_dtms;1943		}1944 1945		arm_cmn_set_index(hw->dtm_idx, i, dtm_idx);1946 1947		dtm->input_sel[dtm_idx] = input_sel;1948		shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx);1949		dtm->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift);1950		dtm->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, hw->dtc_idx[d]) << shift;1951		dtm->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx);1952		reg = (u64)le32_to_cpu(dtm->pmu_config_high) << 32 | dtm->pmu_config_low;1953		writeq_relaxed(reg, dtm->base + CMN_DTM_PMU_CONFIG);1954	}1955 1956	/* Go go go! */1957	arm_cmn_init_counter(event);1958 1959	if (flags & PERF_EF_START)1960		arm_cmn_event_start(event, 0);1961 1962	return 0;1963 1964free_dtms:1965	arm_cmn_event_clear(cmn, event, i);1966	return -ENOSPC;1967}1968 1969static void arm_cmn_event_del(struct perf_event *event, int flags)1970{1971	struct arm_cmn *cmn = to_cmn(event->pmu);1972	struct arm_cmn_hw_event *hw = to_cmn_hw(event);1973	enum cmn_node_type type = CMN_EVENT_TYPE(event);1974 1975	arm_cmn_event_stop(event, PERF_EF_UPDATE);1976 1977	if (type == CMN_TYPE_DTC)1978		cmn->dtc[hw->dtc_idx[0]].cycles = NULL;1979	else1980		arm_cmn_event_clear(cmn, event, hw->num_dns);1981}1982 1983/*1984 * We stop the PMU for both add and read, to avoid skew across DTM counters.1985 * In theory we could use snapshots to read without stopping, but then it1986 * becomes a lot trickier to deal with overlow and racing against interrupts,1987 * plus it seems they don't work properly on some hardware anyway :(1988 */1989static void arm_cmn_start_txn(struct pmu *pmu, unsigned int flags)1990{1991	arm_cmn_set_state(to_cmn(pmu), CMN_STATE_TXN);1992}1993 1994static void arm_cmn_end_txn(struct pmu *pmu)1995{1996	arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_TXN);1997}1998 1999static int arm_cmn_commit_txn(struct pmu *pmu)2000{2001	arm_cmn_end_txn(pmu);2002	return 0;2003}2004 2005static void arm_cmn_migrate(struct arm_cmn *cmn, unsigned int cpu)2006{2007	unsigned int i;2008 2009	perf_pmu_migrate_context(&cmn->pmu, cmn->cpu, cpu);2010	for (i = 0; i < cmn->num_dtcs; i++)2011		irq_set_affinity(cmn->dtc[i].irq, cpumask_of(cpu));2012	cmn->cpu = cpu;2013}2014 2015static int arm_cmn_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)2016{2017	struct arm_cmn *cmn;2018	int node;2019 2020	cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node);2021	node = dev_to_node(cmn->dev);2022	if (cpu_to_node(cmn->cpu) != node && cpu_to_node(cpu) == node)2023		arm_cmn_migrate(cmn, cpu);2024	return 0;2025}2026 2027static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)2028{2029	struct arm_cmn *cmn;2030	unsigned int target;2031	int node;2032 2033	cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node);2034	if (cpu != cmn->cpu)2035		return 0;2036 2037	node = dev_to_node(cmn->dev);2038 2039	target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);2040	if (target >= nr_cpu_ids)2041		target = cpumask_any_but(cpu_online_mask, cpu);2042 2043	if (target < nr_cpu_ids)2044		arm_cmn_migrate(cmn, target);2045 2046	return 0;2047}2048 2049static irqreturn_t arm_cmn_handle_irq(int irq, void *dev_id)2050{2051	struct arm_cmn_dtc *dtc = dev_id;2052	irqreturn_t ret = IRQ_NONE;2053 2054	for (;;) {2055		u32 status = readl_relaxed(CMN_DT_PMOVSR(dtc));2056		u64 delta;2057		int i;2058 2059		for (i = 0; i < CMN_DT_NUM_COUNTERS; i++) {2060			if (status & (1U << i)) {2061				ret = IRQ_HANDLED;2062				if (WARN_ON(!dtc->counters[i]))2063					continue;2064				delta = (u64)arm_cmn_read_counter(dtc, i) << 16;2065				local64_add(delta, &dtc->counters[i]->count);2066			}2067		}2068 2069		if (status & (1U << CMN_DT_NUM_COUNTERS)) {2070			ret = IRQ_HANDLED;2071			if (dtc->cc_active && !WARN_ON(!dtc->cycles)) {2072				delta = arm_cmn_read_cc(dtc);2073				local64_add(delta, &dtc->cycles->count);2074			}2075		}2076 2077		writel_relaxed(status, CMN_DT_PMOVSR_CLR(dtc));2078 2079		if (!dtc->irq_friend)2080			return ret;2081		dtc += dtc->irq_friend;2082	}2083}2084 2085/* We can reasonably accommodate DTCs of the same CMN sharing IRQs */2086static int arm_cmn_init_irqs(struct arm_cmn *cmn)2087{2088	int i, j, irq, err;2089 2090	for (i = 0; i < cmn->num_dtcs; i++) {2091		irq = cmn->dtc[i].irq;2092		for (j = i; j--; ) {2093			if (cmn->dtc[j].irq == irq) {2094				cmn->dtc[j].irq_friend = i - j;2095				goto next;2096			}2097		}2098		err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq,2099				       IRQF_NOBALANCING | IRQF_NO_THREAD,2100				       dev_name(cmn->dev), &cmn->dtc[i]);2101		if (err)2102			return err;2103 2104		err = irq_set_affinity(irq, cpumask_of(cmn->cpu));2105		if (err)2106			return err;2107	next:2108		; /* isn't C great? */2109	}2110	return 0;2111}2112 2113static void arm_cmn_init_dtm(struct arm_cmn_dtm *dtm, struct arm_cmn_node *xp, int idx)2114{2115	int i;2116 2117	dtm->base = xp->pmu_base + CMN_DTM_OFFSET(idx);2118	dtm->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN;2119	writeq_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG);2120	for (i = 0; i < 4; i++) {2121		dtm->wp_event[i] = -1;2122		writeq_relaxed(0, dtm->base + CMN_DTM_WPn_MASK(i));2123		writeq_relaxed(~0ULL, dtm->base + CMN_DTM_WPn_VAL(i));2124	}2125}2126 2127static int arm_cmn_init_dtc(struct arm_cmn *cmn, struct arm_cmn_node *dn, int idx)2128{2129	struct arm_cmn_dtc *dtc = cmn->dtc + idx;2130 2131	dtc->pmu_base = dn->pmu_base;2132	dtc->base = dtc->pmu_base - arm_cmn_pmu_offset(cmn, dn);2133	dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx);2134	if (dtc->irq < 0)2135		return dtc->irq;2136 2137	writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL);2138	writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN, CMN_DT_PMCR(dtc));2139	writeq_relaxed(0, CMN_DT_PMCCNTR(dtc));2140	writel_relaxed(0x1ff, CMN_DT_PMOVSR_CLR(dtc));2141 2142	return 0;2143}2144 2145static int arm_cmn_node_cmp(const void *a, const void *b)2146{2147	const struct arm_cmn_node *dna = a, *dnb = b;2148	int cmp;2149 2150	cmp = dna->type - dnb->type;2151	if (!cmp)2152		cmp = dna->logid - dnb->logid;2153	return cmp;2154}2155 2156static int arm_cmn_init_dtcs(struct arm_cmn *cmn)2157{2158	struct arm_cmn_node *dn, *xp;2159	int dtc_idx = 0;2160 2161	cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL);2162	if (!cmn->dtc)2163		return -ENOMEM;2164 2165	sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL);2166 2167	cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP);2168 2169	if (cmn->part == PART_CMN600 && cmn->num_dtcs > 1) {2170		/* We do at least know that a DTC's XP must be in that DTC's domain */2171		dn = arm_cmn_node(cmn, CMN_TYPE_DTC);2172		for (int i = 0; i < cmn->num_dtcs; i++)2173			arm_cmn_node_to_xp(cmn, dn + i)->dtc = i;2174	}2175 2176	for (dn = cmn->dns; dn->type; dn++) {2177		if (dn->type == CMN_TYPE_XP)2178			continue;2179 2180		xp = arm_cmn_node_to_xp(cmn, dn);2181		dn->portid_bits = xp->portid_bits;2182		dn->deviceid_bits = xp->deviceid_bits;2183		dn->dtc = xp->dtc;2184		dn->dtm = xp->dtm;2185		if (cmn->multi_dtm)2186			dn->dtm += arm_cmn_nid(dn).port / 2;2187 2188		if (dn->type == CMN_TYPE_DTC) {2189			int err = arm_cmn_init_dtc(cmn, dn, dtc_idx++);2190 2191			if (err)2192				return err;2193		}2194 2195		/* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */2196		if (dn->type == CMN_TYPE_RND)2197			dn->type = CMN_TYPE_RNI;2198 2199		/* We split the RN-I off already, so let the CCLA part match CCLA events */2200		if (dn->type == CMN_TYPE_CCLA_RNI)2201			dn->type = CMN_TYPE_CCLA;2202	}2203 2204	arm_cmn_set_state(cmn, CMN_STATE_DISABLED);2205 2206	return 0;2207}2208 2209static unsigned int arm_cmn_dtc_domain(struct arm_cmn *cmn, void __iomem *xp_region)2210{2211	int offset = CMN_DTM_UNIT_INFO;2212 2213	if (cmn->part == PART_CMN650 || cmn->part == PART_CI700)2214		offset = CMN650_DTM_UNIT_INFO;2215 2216	return FIELD_GET(CMN_DTM_UNIT_INFO_DTC_DOMAIN, readl_relaxed(xp_region + offset));2217}2218 2219static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node)2220{2221	int level;2222	u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO);2223 2224	node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg);2225	node->id = FIELD_GET(CMN_NI_NODE_ID, reg);2226	node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg);2227 2228	node->pmu_base = cmn->base + offset + arm_cmn_pmu_offset(cmn, node);2229 2230	if (node->type == CMN_TYPE_CFG)2231		level = 0;2232	else if (node->type == CMN_TYPE_XP)2233		level = 1;2234	else2235		level = 2;2236 2237	dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n",2238			(level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ',2239			node->type, node->logid, offset);2240}2241 2242static enum cmn_node_type arm_cmn_subtype(enum cmn_node_type type)2243{2244	switch (type) {2245	case CMN_TYPE_HNP:2246		return CMN_TYPE_HNI;2247	case CMN_TYPE_CCLA_RNI:2248		return CMN_TYPE_RNI;2249	default:2250		return CMN_TYPE_INVALID;2251	}2252}2253 2254static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)2255{2256	void __iomem *cfg_region;2257	struct arm_cmn_node cfg, *dn;2258	struct arm_cmn_dtm *dtm;2259	enum cmn_part part;2260	u16 child_count, child_poff;2261	u32 xp_offset[CMN_MAX_XPS];2262	u64 reg;2263	int i, j;2264	size_t sz;2265 2266	arm_cmn_init_node_info(cmn, rgn_offset, &cfg);2267	if (cfg.type != CMN_TYPE_CFG)2268		return -ENODEV;2269 2270	cfg_region = cmn->base + rgn_offset;2271 2272	reg = readq_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_01);2273	part = FIELD_GET(CMN_CFGM_PID0_PART_0, reg);2274	part |= FIELD_GET(CMN_CFGM_PID1_PART_1, reg) << 8;2275	if (cmn->part && cmn->part != part)2276		dev_warn(cmn->dev,2277			 "Firmware binding mismatch: expected part number 0x%x, found 0x%x\n",2278			 cmn->part, part);2279	cmn->part = part;2280	if (!arm_cmn_model(cmn))2281		dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part);2282 2283	reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_23);2284	cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg);2285 2286	/*2287	 * With the device isolation feature, if firmware has neglected to enable2288	 * an XP port then we risk locking up if we try to access anything behind2289	 * it; however we also have no way to tell from Non-Secure whether any2290	 * given port is disabled or not, so the only way to win is not to play...2291	 */2292	reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL);2293	if (reg & CMN_INFO_DEVICE_ISO_ENABLE) {2294		dev_err(cmn->dev, "Device isolation enabled, not continuing due to risk of lockup\n");2295		return -ENODEV;2296	}2297	cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN;2298	cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg);2299	cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg);2300 2301	reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL_1);2302	cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg);2303	cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg);2304 2305	reg = readq_relaxed(cfg_region + CMN_CHILD_INFO);2306	child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);2307	child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);2308 2309	cmn->num_xps = child_count;2310	cmn->num_dns = cmn->num_xps;2311 2312	/* Pass 1: visit the XPs, enumerate their children */2313	for (i = 0; i < cmn->num_xps; i++) {2314		reg = readq_relaxed(cfg_region + child_poff + i * 8);2315		xp_offset[i] = reg & CMN_CHILD_NODE_ADDR;2316 2317		reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO);2318		cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg);2319	}2320 2321	/*2322	 * Some nodes effectively have two separate types, which we'll handle2323	 * by creating one of each internally. For a (very) safe initial upper2324	 * bound, account for double the number of non-XP nodes.2325	 */2326	dn = devm_kcalloc(cmn->dev, cmn->num_dns * 2 - cmn->num_xps,2327			  sizeof(*dn), GFP_KERNEL);2328	if (!dn)2329		return -ENOMEM;2330 2331	/* Initial safe upper bound on DTMs for any possible mesh layout */2332	i = cmn->num_xps;2333	if (cmn->multi_dtm)2334		i += cmn->num_xps + 1;2335	dtm = devm_kcalloc(cmn->dev, i, sizeof(*dtm), GFP_KERNEL);2336	if (!dtm)2337		return -ENOMEM;2338 2339	/* Pass 2: now we can actually populate the nodes */2340	cmn->dns = dn;2341	cmn->dtms = dtm;2342	for (i = 0; i < cmn->num_xps; i++) {2343		void __iomem *xp_region = cmn->base + xp_offset[i];2344		struct arm_cmn_node *xp = dn++;2345		unsigned int xp_ports = 0;2346 2347		arm_cmn_init_node_info(cmn, xp_offset[i], xp);2348		/*2349		 * Thanks to the order in which XP logical IDs seem to be2350		 * assigned, we can handily infer the mesh X dimension by2351		 * looking out for the XP at (0,1) without needing to know2352		 * the exact node ID format, which we can later derive.2353		 */2354		if (xp->id == (1 << 3))2355			cmn->mesh_x = xp->logid;2356 2357		if (cmn->part == PART_CMN600)2358			xp->dtc = -1;2359		else2360			xp->dtc = arm_cmn_dtc_domain(cmn, xp_region);2361 2362		xp->dtm = dtm - cmn->dtms;2363		arm_cmn_init_dtm(dtm++, xp, 0);2364		/*2365		 * Keeping track of connected ports will let us filter out2366		 * unnecessary XP events easily, and also infer the per-XP2367		 * part of the node ID format.2368		 */2369		for (int p = 0; p < CMN_MAX_PORTS; p++)2370			if (arm_cmn_device_connect_info(cmn, xp, p))2371				xp_ports |= BIT(p);2372 2373		if (cmn->num_xps == 1) {2374			xp->portid_bits = 3;2375			xp->deviceid_bits = 2;2376		} else if (xp_ports > 0x3) {2377			xp->portid_bits = 2;2378			xp->deviceid_bits = 1;2379		} else {2380			xp->portid_bits = 1;2381			xp->deviceid_bits = 2;2382		}2383 2384		if (cmn->multi_dtm && (xp_ports > 0x3))2385			arm_cmn_init_dtm(dtm++, xp, 1);2386		if (cmn->multi_dtm && (xp_ports > 0xf))2387			arm_cmn_init_dtm(dtm++, xp, 2);2388 2389		cmn->ports_used |= xp_ports;2390 2391		reg = readq_relaxed(xp_region + CMN_CHILD_INFO);2392		child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);2393		child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);2394 2395		for (j = 0; j < child_count; j++) {2396			reg = readq_relaxed(xp_region + child_poff + j * 8);2397			/*2398			 * Don't even try to touch anything external, since in general2399			 * we haven't a clue how to power up arbitrary CHI requesters.2400			 * As of CMN-600r1 these could only be RN-SAMs or CXLAs,2401			 * neither of which have any PMU events anyway.2402			 * (Actually, CXLAs do seem to have grown some events in r1p2,2403			 * but they don't go to regular XP DTMs, and they depend on2404			 * secure configuration which we can't easily deal with)2405			 */2406			if (reg & CMN_CHILD_NODE_EXTERNAL) {2407				dev_dbg(cmn->dev, "ignoring external node %llx\n", reg);2408				continue;2409			}2410			/*2411			 * AmpereOneX erratum AC04_MESH_1 makes some XPs report a bogus2412			 * child count larger than the number of valid child pointers.2413			 * A child offset of 0 can only occur on CMN-600; otherwise it2414			 * would imply the root node being its own grandchild, which2415			 * we can safely dismiss in general.2416			 */2417			if (reg == 0 && cmn->part != PART_CMN600) {2418				dev_dbg(cmn->dev, "bogus child pointer?\n");2419				continue;2420			}2421 2422			arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn);2423 2424			switch (dn->type) {2425			case CMN_TYPE_DTC:2426				cmn->num_dtcs++;2427				dn++;2428				break;2429			/* These guys have PMU events */2430			case CMN_TYPE_DVM:2431			case CMN_TYPE_HNI:2432			case CMN_TYPE_HNF:2433			case CMN_TYPE_SBSX:2434			case CMN_TYPE_RNI:2435			case CMN_TYPE_RND:2436			case CMN_TYPE_MTSX:2437			case CMN_TYPE_CXRA:2438			case CMN_TYPE_CXHA:2439			case CMN_TYPE_CCRA:2440			case CMN_TYPE_CCHA:2441			case CMN_TYPE_HNS:2442				dn++;2443				break;2444			case CMN_TYPE_CCLA:2445				dn->pmu_base += CMN_CCLA_PMU_EVENT_SEL;2446				dn++;2447				break;2448			/* Nothing to see here */2449			case CMN_TYPE_MPAM_S:2450			case CMN_TYPE_MPAM_NS:2451			case CMN_TYPE_RNSAM:2452			case CMN_TYPE_CXLA:2453			case CMN_TYPE_HNS_MPAM_S:2454			case CMN_TYPE_HNS_MPAM_NS:2455			case CMN_TYPE_APB:2456				break;2457			/*2458			 * Split "optimised" combination nodes into separate2459			 * types for the different event sets. Offsetting the2460			 * base address lets us handle the second pmu_event_sel2461			 * register via the normal mechanism later.2462			 */2463			case CMN_TYPE_HNP:2464			case CMN_TYPE_CCLA_RNI:2465				dn[1] = dn[0];2466				dn[0].pmu_base += CMN_CCLA_PMU_EVENT_SEL;2467				dn[1].type = arm_cmn_subtype(dn->type);2468				dn += 2;2469				break;2470			/* Something has gone horribly wrong */2471			default:2472				dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type);2473				return -ENODEV;2474			}2475		}2476	}2477 2478	/* Correct for any nodes we added or skipped */2479	cmn->num_dns = dn - cmn->dns;2480 2481	/* Cheeky +1 to help terminate pointer-based iteration later */2482	sz = (void *)(dn + 1) - (void *)cmn->dns;2483	dn = devm_krealloc(cmn->dev, cmn->dns, sz, GFP_KERNEL);2484	if (dn)2485		cmn->dns = dn;2486 2487	sz = (void *)dtm - (void *)cmn->dtms;2488	dtm = devm_krealloc(cmn->dev, cmn->dtms, sz, GFP_KERNEL);2489	if (dtm)2490		cmn->dtms = dtm;2491 2492	/*2493	 * If mesh_x wasn't set during discovery then we never saw2494	 * an XP at (0,1), thus we must have an Nx1 configuration.2495	 */2496	if (!cmn->mesh_x)2497		cmn->mesh_x = cmn->num_xps;2498	cmn->mesh_y = cmn->num_xps / cmn->mesh_x;2499 2500	/* 1x1 config plays havoc with XP event encodings */2501	if (cmn->num_xps == 1)2502		dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n");2503 2504	dev_dbg(cmn->dev, "periph_id part 0x%03x revision %d\n", cmn->part, cmn->rev);2505	reg = cmn->ports_used;2506	dev_dbg(cmn->dev, "mesh %dx%d, ID width %d, ports %6pbl%s\n",2507		cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn), &reg,2508		cmn->multi_dtm ? ", multi-DTM" : "");2509 2510	return 0;2511}2512 2513static int arm_cmn600_acpi_probe(struct platform_device *pdev, struct arm_cmn *cmn)2514{2515	struct resource *cfg, *root;2516 2517	cfg = platform_get_resource(pdev, IORESOURCE_MEM, 0);2518	if (!cfg)2519		return -EINVAL;2520 2521	root = platform_get_resource(pdev, IORESOURCE_MEM, 1);2522	if (!root)2523		return -EINVAL;2524 2525	if (!resource_contains(cfg, root))2526		swap(cfg, root);2527	/*2528	 * Note that devm_ioremap_resource() is dumb and won't let the platform2529	 * device claim cfg when the ACPI companion device has already claimed2530	 * root within it. But since they *are* already both claimed in the2531	 * appropriate name, we don't really need to do it again here anyway.2532	 */2533	cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg));2534	if (!cmn->base)2535		return -ENOMEM;2536 2537	return root->start - cfg->start;2538}2539 2540static int arm_cmn600_of_probe(struct device_node *np)2541{2542	u32 rootnode;2543 2544	return of_property_read_u32(np, "arm,root-node", &rootnode) ?: rootnode;2545}2546 2547static int arm_cmn_probe(struct platform_device *pdev)2548{2549	struct arm_cmn *cmn;2550	const char *name;2551	static atomic_t id;2552	int err, rootnode, this_id;2553 2554	cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL);2555	if (!cmn)2556		return -ENOMEM;2557 2558	cmn->dev = &pdev->dev;2559	cmn->part = (unsigned long)device_get_match_data(cmn->dev);2560	platform_set_drvdata(pdev, cmn);2561 2562	if (cmn->part == PART_CMN600 && has_acpi_companion(cmn->dev)) {2563		rootnode = arm_cmn600_acpi_probe(pdev, cmn);2564	} else {2565		rootnode = 0;2566		cmn->base = devm_platform_ioremap_resource(pdev, 0);2567		if (IS_ERR(cmn->base))2568			return PTR_ERR(cmn->base);2569		if (cmn->part == PART_CMN600)2570			rootnode = arm_cmn600_of_probe(pdev->dev.of_node);2571	}2572	if (rootnode < 0)2573		return rootnode;2574 2575	err = arm_cmn_discover(cmn, rootnode);2576	if (err)2577		return err;2578 2579	err = arm_cmn_init_dtcs(cmn);2580	if (err)2581		return err;2582 2583	err = arm_cmn_init_irqs(cmn);2584	if (err)2585		return err;2586 2587	cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev));2588	cmn->pmu = (struct pmu) {2589		.module = THIS_MODULE,2590		.parent = cmn->dev,2591		.attr_groups = arm_cmn_attr_groups,2592		.capabilities = PERF_PMU_CAP_NO_EXCLUDE,2593		.task_ctx_nr = perf_invalid_context,2594		.pmu_enable = arm_cmn_pmu_enable,2595		.pmu_disable = arm_cmn_pmu_disable,2596		.event_init = arm_cmn_event_init,2597		.add = arm_cmn_event_add,2598		.del = arm_cmn_event_del,2599		.start = arm_cmn_event_start,2600		.stop = arm_cmn_event_stop,2601		.read = arm_cmn_event_read,2602		.start_txn = arm_cmn_start_txn,2603		.commit_txn = arm_cmn_commit_txn,2604		.cancel_txn = arm_cmn_end_txn,2605	};2606 2607	this_id = atomic_fetch_inc(&id);2608	name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id);2609	if (!name)2610		return -ENOMEM;2611 2612	err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node);2613	if (err)2614		return err;2615 2616	err = perf_pmu_register(&cmn->pmu, name, -1);2617	if (err)2618		cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node);2619	else2620		arm_cmn_debugfs_init(cmn, this_id);2621 2622	return err;2623}2624 2625static void arm_cmn_remove(struct platform_device *pdev)2626{2627	struct arm_cmn *cmn = platform_get_drvdata(pdev);2628 2629	writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL);2630 2631	perf_pmu_unregister(&cmn->pmu);2632	cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node);2633	debugfs_remove(cmn->debug);2634}2635 2636#ifdef CONFIG_OF2637static const struct of_device_id arm_cmn_of_match[] = {2638	{ .compatible = "arm,cmn-600", .data = (void *)PART_CMN600 },2639	{ .compatible = "arm,cmn-650" },2640	{ .compatible = "arm,cmn-700" },2641	{ .compatible = "arm,cmn-s3" },2642	{ .compatible = "arm,ci-700" },2643	{}2644};2645MODULE_DEVICE_TABLE(of, arm_cmn_of_match);2646#endif2647 2648#ifdef CONFIG_ACPI2649static const struct acpi_device_id arm_cmn_acpi_match[] = {2650	{ "ARMHC600", PART_CMN600 },2651	{ "ARMHC650" },2652	{ "ARMHC700" },2653	{}2654};2655MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match);2656#endif2657 2658static struct platform_driver arm_cmn_driver = {2659	.driver = {2660		.name = "arm-cmn",2661		.of_match_table = of_match_ptr(arm_cmn_of_match),2662		.acpi_match_table = ACPI_PTR(arm_cmn_acpi_match),2663	},2664	.probe = arm_cmn_probe,2665	.remove_new = arm_cmn_remove,2666};2667 2668static int __init arm_cmn_init(void)2669{2670	int ret;2671 2672	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,2673				      "perf/arm/cmn:online",2674				      arm_cmn_pmu_online_cpu,2675				      arm_cmn_pmu_offline_cpu);2676	if (ret < 0)2677		return ret;2678 2679	arm_cmn_hp_state = ret;2680	arm_cmn_debugfs = debugfs_create_dir("arm-cmn", NULL);2681 2682	ret = platform_driver_register(&arm_cmn_driver);2683	if (ret) {2684		cpuhp_remove_multi_state(arm_cmn_hp_state);2685		debugfs_remove(arm_cmn_debugfs);2686	}2687	return ret;2688}2689 2690static void __exit arm_cmn_exit(void)2691{2692	platform_driver_unregister(&arm_cmn_driver);2693	cpuhp_remove_multi_state(arm_cmn_hp_state);2694	debugfs_remove(arm_cmn_debugfs);2695}2696 2697module_init(arm_cmn_init);2698module_exit(arm_cmn_exit);2699 2700MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");2701MODULE_DESCRIPTION("Arm CMN-600 PMU driver");2702MODULE_LICENSE("GPL v2");2703