858 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright 2023 NXP3 4#include <linux/bitfield.h>5#include <linux/init.h>6#include <linux/interrupt.h>7#include <linux/io.h>8#include <linux/module.h>9#include <linux/of.h>10#include <linux/platform_device.h>11#include <linux/perf_event.h>12 13/* Performance monitor configuration */14#define PMCFG1 0x0015#define MX93_PMCFG1_RD_TRANS_FILT_EN BIT(31)16#define MX93_PMCFG1_WR_TRANS_FILT_EN BIT(30)17#define MX93_PMCFG1_RD_BT_FILT_EN BIT(29)18#define MX93_PMCFG1_ID_MASK GENMASK(17, 0)19 20#define MX95_PMCFG1_WR_BEAT_FILT_EN BIT(31)21#define MX95_PMCFG1_RD_BEAT_FILT_EN BIT(30)22 23#define PMCFG2 0x0424#define MX93_PMCFG2_ID GENMASK(17, 0)25 26#define PMCFG3 0x0827#define PMCFG4 0x0C28#define PMCFG5 0x1029#define PMCFG6 0x1430#define MX95_PMCFG_ID_MASK GENMASK(9, 0)31#define MX95_PMCFG_ID GENMASK(25, 16)32 33/* Global control register affects all counters and takes priority over local control registers */34#define PMGC0 0x4035/* Global control register bits */36#define PMGC0_FAC BIT(31)37#define PMGC0_PMIE BIT(30)38#define PMGC0_FCECE BIT(29)39 40/*41 * 64bit counter0 exclusively dedicated to counting cycles42 * 32bit counters monitor counter-specific events in addition to counting reference events43 */44#define PMLCA(n) (0x40 + 0x10 + (0x10 * n))45#define PMLCB(n) (0x40 + 0x14 + (0x10 * n))46#define PMC(n) (0x40 + 0x18 + (0x10 * n))47/* Local control register bits */48#define PMLCA_FC BIT(31)49#define PMLCA_CE BIT(26)50#define PMLCA_EVENT GENMASK(22, 16)51 52#define NUM_COUNTERS 1153#define CYCLES_COUNTER 054#define CYCLES_EVENT_ID 055 56#define CONFIG_EVENT_MASK GENMASK(7, 0)57#define CONFIG_COUNTER_MASK GENMASK(23, 16)58 59#define to_ddr_pmu(p) container_of(p, struct ddr_pmu, pmu)60 61#define DDR_PERF_DEV_NAME "imx9_ddr"62#define DDR_CPUHP_CB_NAME DDR_PERF_DEV_NAME "_perf_pmu"63 64static DEFINE_IDA(ddr_ida);65 66struct imx_ddr_devtype_data {67 const char *identifier; /* system PMU identifier for userspace */68};69 70struct ddr_pmu {71 struct pmu pmu;72 void __iomem *base;73 unsigned int cpu;74 struct hlist_node node;75 struct device *dev;76 struct perf_event *events[NUM_COUNTERS];77 int active_events;78 enum cpuhp_state cpuhp_state;79 const struct imx_ddr_devtype_data *devtype_data;80 int irq;81 int id;82};83 84static const struct imx_ddr_devtype_data imx93_devtype_data = {85 .identifier = "imx93",86};87 88static const struct imx_ddr_devtype_data imx95_devtype_data = {89 .identifier = "imx95",90};91 92static inline bool is_imx93(struct ddr_pmu *pmu)93{94 return pmu->devtype_data == &imx93_devtype_data;95}96 97static inline bool is_imx95(struct ddr_pmu *pmu)98{99 return pmu->devtype_data == &imx95_devtype_data;100}101 102static const struct of_device_id imx_ddr_pmu_dt_ids[] = {103 { .compatible = "fsl,imx93-ddr-pmu", .data = &imx93_devtype_data },104 { .compatible = "fsl,imx95-ddr-pmu", .data = &imx95_devtype_data },105 { /* sentinel */ }106};107MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids);108 109static ssize_t ddr_perf_identifier_show(struct device *dev,110 struct device_attribute *attr,111 char *page)112{113 struct ddr_pmu *pmu = dev_get_drvdata(dev);114 115 return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);116}117 118static struct device_attribute ddr_perf_identifier_attr =119 __ATTR(identifier, 0444, ddr_perf_identifier_show, NULL);120 121static struct attribute *ddr_perf_identifier_attrs[] = {122 &ddr_perf_identifier_attr.attr,123 NULL,124};125 126static struct attribute_group ddr_perf_identifier_attr_group = {127 .attrs = ddr_perf_identifier_attrs,128};129 130static ssize_t ddr_perf_cpumask_show(struct device *dev,131 struct device_attribute *attr, char *buf)132{133 struct ddr_pmu *pmu = dev_get_drvdata(dev);134 135 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));136}137 138static struct device_attribute ddr_perf_cpumask_attr =139 __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL);140 141static struct attribute *ddr_perf_cpumask_attrs[] = {142 &ddr_perf_cpumask_attr.attr,143 NULL,144};145 146static const struct attribute_group ddr_perf_cpumask_attr_group = {147 .attrs = ddr_perf_cpumask_attrs,148};149 150struct imx9_pmu_events_attr {151 struct device_attribute attr;152 u64 id;153 const void *devtype_data;154};155 156static ssize_t ddr_pmu_event_show(struct device *dev,157 struct device_attribute *attr, char *page)158{159 struct imx9_pmu_events_attr *pmu_attr;160 161 pmu_attr = container_of(attr, struct imx9_pmu_events_attr, attr);162 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);163}164 165#define COUNTER_OFFSET_IN_EVENT 8166#define ID(counter, id) ((counter << COUNTER_OFFSET_IN_EVENT) | id)167 168#define DDR_PMU_EVENT_ATTR_COMM(_name, _id, _data) \169 (&((struct imx9_pmu_events_attr[]) { \170 { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\171 .id = _id, \172 .devtype_data = _data, } \173 })[0].attr.attr)174 175#define IMX9_DDR_PMU_EVENT_ATTR(_name, _id) \176 DDR_PMU_EVENT_ATTR_COMM(_name, _id, NULL)177 178#define IMX93_DDR_PMU_EVENT_ATTR(_name, _id) \179 DDR_PMU_EVENT_ATTR_COMM(_name, _id, &imx93_devtype_data)180 181#define IMX95_DDR_PMU_EVENT_ATTR(_name, _id) \182 DDR_PMU_EVENT_ATTR_COMM(_name, _id, &imx95_devtype_data)183 184static struct attribute *ddr_perf_events_attrs[] = {185 /* counter0 cycles event */186 IMX9_DDR_PMU_EVENT_ATTR(cycles, 0),187 188 /* reference events for all normal counters, need assert DEBUG19[21] bit */189 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ddrc1_rmw_for_ecc, 12),190 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_rreorder, 13),191 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_wreorder, 14),192 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_0, 15),193 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_1, 16),194 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_2, 17),195 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_3, 18),196 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_4, 19),197 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_5, 22),198 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_6, 23),199 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_7, 24),200 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_8, 25),201 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_9, 26),202 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_10, 27),203 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_11, 28),204 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_12, 31),205 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_13, 59),206 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_15, 61),207 IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_29, 63),208 209 /* counter1 specific events */210 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_0, ID(1, 64)),211 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_1, ID(1, 65)),212 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_2, ID(1, 66)),213 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_3, ID(1, 67)),214 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_4, ID(1, 68)),215 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_5, ID(1, 69)),216 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_6, ID(1, 70)),217 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_7, ID(1, 71)),218 219 /* counter2 specific events */220 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_0, ID(2, 64)),221 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_1, ID(2, 65)),222 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_2, ID(2, 66)),223 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_3, ID(2, 67)),224 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_4, ID(2, 68)),225 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_5, ID(2, 69)),226 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_6, ID(2, 70)),227 IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_7, ID(2, 71)),228 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_empty, ID(2, 72)),229 IMX93_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_trans_filt, ID(2, 73)), /* imx93 specific*/230 IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_wr_beat_filt, ID(2, 73)), /* imx95 specific*/231 232 /* counter3 specific events */233 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_0, ID(3, 64)),234 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_1, ID(3, 65)),235 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_2, ID(3, 66)),236 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_3, ID(3, 67)),237 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_4, ID(3, 68)),238 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_5, ID(3, 69)),239 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_6, ID(3, 70)),240 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_7, ID(3, 71)),241 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_full, ID(3, 72)),242 IMX93_DDR_PMU_EVENT_ATTR(eddrtq_pm_wr_trans_filt, ID(3, 73)), /* imx93 specific*/243 IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt2, ID(3, 73)), /* imx95 specific*/244 245 /* counter4 specific events */246 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_0, ID(4, 64)),247 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_1, ID(4, 65)),248 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_2, ID(4, 66)),249 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_3, ID(4, 67)),250 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_4, ID(4, 68)),251 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_5, ID(4, 69)),252 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_6, ID(4, 70)),253 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_7, ID(4, 71)),254 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2_rmw, ID(4, 72)),255 IMX93_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt, ID(4, 73)), /* imx93 specific*/256 IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt1, ID(4, 73)), /* imx95 specific*/257 258 /* counter5 specific events */259 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_0, ID(5, 64)),260 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_1, ID(5, 65)),261 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_2, ID(5, 66)),262 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_3, ID(5, 67)),263 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_4, ID(5, 68)),264 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_5, ID(5, 69)),265 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_6, ID(5, 70)),266 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_7, ID(5, 71)),267 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq1, ID(5, 72)),268 IMX95_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt0, ID(5, 73)), /* imx95 specific*/269 270 /* counter6 specific events */271 IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_end_0, ID(6, 64)),272 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2, ID(6, 72)),273 274 /* counter7 specific events */275 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_2_full, ID(7, 64)),276 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq0, ID(7, 65)),277 278 /* counter8 specific events */279 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_bias_switched, ID(8, 64)),280 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_4_full, ID(8, 65)),281 282 /* counter9 specific events */283 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq1, ID(9, 65)),284 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_3_4_full, ID(9, 66)),285 286 /* counter10 specific events */287 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_misc_mrk, ID(10, 65)),288 IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq0, ID(10, 66)),289 NULL,290};291 292static umode_t293ddr_perf_events_attrs_is_visible(struct kobject *kobj,294 struct attribute *attr, int unused)295{296 struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));297 struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);298 struct imx9_pmu_events_attr *eattr;299 300 eattr = container_of(attr, typeof(*eattr), attr.attr);301 302 if (!eattr->devtype_data)303 return attr->mode;304 305 if (eattr->devtype_data != ddr_pmu->devtype_data)306 return 0;307 308 return attr->mode;309}310 311static const struct attribute_group ddr_perf_events_attr_group = {312 .name = "events",313 .attrs = ddr_perf_events_attrs,314 .is_visible = ddr_perf_events_attrs_is_visible,315};316 317PMU_FORMAT_ATTR(event, "config:0-7,16-23");318PMU_FORMAT_ATTR(counter, "config:8-15");319PMU_FORMAT_ATTR(axi_id, "config1:0-17");320PMU_FORMAT_ATTR(axi_mask, "config2:0-17");321 322static struct attribute *ddr_perf_format_attrs[] = {323 &format_attr_event.attr,324 &format_attr_counter.attr,325 &format_attr_axi_id.attr,326 &format_attr_axi_mask.attr,327 NULL,328};329 330static const struct attribute_group ddr_perf_format_attr_group = {331 .name = "format",332 .attrs = ddr_perf_format_attrs,333};334 335static const struct attribute_group *attr_groups[] = {336 &ddr_perf_identifier_attr_group,337 &ddr_perf_cpumask_attr_group,338 &ddr_perf_events_attr_group,339 &ddr_perf_format_attr_group,340 NULL,341};342 343static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter)344{345 if (counter == CYCLES_COUNTER) {346 writel(0, pmu->base + PMC(counter) + 0x4);347 writel(0, pmu->base + PMC(counter));348 } else {349 writel(0, pmu->base + PMC(counter));350 }351}352 353static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)354{355 u32 val_lower, val_upper;356 u64 val;357 358 if (counter != CYCLES_COUNTER) {359 val = readl_relaxed(pmu->base + PMC(counter));360 goto out;361 }362 363 /* special handling for reading 64bit cycle counter */364 do {365 val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4);366 val_lower = readl_relaxed(pmu->base + PMC(counter));367 } while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4));368 369 val = val_upper;370 val = (val << 32);371 val |= val_lower;372out:373 return val;374}375 376static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable)377{378 u32 ctrl;379 380 ctrl = readl_relaxed(pmu->base + PMGC0);381 382 if (enable) {383 /*384 * The performance monitor must be reset before event counting385 * sequences. The performance monitor can be reset by first freezing386 * one or more counters and then clearing the freeze condition to387 * allow the counters to count according to the settings in the388 * performance monitor registers. Counters can be frozen individually389 * by setting PMLCAn[FC] bits, or simultaneously by setting PMGC0[FAC].390 * Simply clearing these freeze bits will then allow the performance391 * monitor to begin counting based on the register settings.392 */393 ctrl |= PMGC0_FAC;394 writel(ctrl, pmu->base + PMGC0);395 396 /*397 * Freeze all counters disabled, interrupt enabled, and freeze398 * counters on condition enabled.399 */400 ctrl &= ~PMGC0_FAC;401 ctrl |= PMGC0_PMIE | PMGC0_FCECE;402 writel(ctrl, pmu->base + PMGC0);403 } else {404 ctrl |= PMGC0_FAC;405 ctrl &= ~(PMGC0_PMIE | PMGC0_FCECE);406 writel(ctrl, pmu->base + PMGC0);407 }408}409 410static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config,411 int counter, bool enable)412{413 u32 ctrl_a;414 int event;415 416 ctrl_a = readl_relaxed(pmu->base + PMLCA(counter));417 event = FIELD_GET(CONFIG_EVENT_MASK, config);418 419 if (enable) {420 ctrl_a |= PMLCA_FC;421 writel(ctrl_a, pmu->base + PMLCA(counter));422 423 ddr_perf_clear_counter(pmu, counter);424 425 /* Freeze counter disabled, condition enabled, and program event.*/426 ctrl_a &= ~PMLCA_FC;427 ctrl_a |= PMLCA_CE;428 ctrl_a &= ~FIELD_PREP(PMLCA_EVENT, 0x7F);429 ctrl_a |= FIELD_PREP(PMLCA_EVENT, event);430 writel(ctrl_a, pmu->base + PMLCA(counter));431 } else {432 /* Freeze counter. */433 ctrl_a |= PMLCA_FC;434 writel(ctrl_a, pmu->base + PMLCA(counter));435 }436}437 438static void imx93_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,439 int counter, int axi_id, int axi_mask)440{441 u32 pmcfg1, pmcfg2;442 u32 mask[] = { MX93_PMCFG1_RD_TRANS_FILT_EN,443 MX93_PMCFG1_WR_TRANS_FILT_EN,444 MX93_PMCFG1_RD_BT_FILT_EN };445 446 pmcfg1 = readl_relaxed(pmu->base + PMCFG1);447 448 if (counter >= 2 && counter <= 4)449 pmcfg1 = event == 73 ? pmcfg1 | mask[counter - 2] :450 pmcfg1 & ~mask[counter - 2];451 452 pmcfg1 &= ~FIELD_PREP(MX93_PMCFG1_ID_MASK, 0x3FFFF);453 pmcfg1 |= FIELD_PREP(MX93_PMCFG1_ID_MASK, axi_mask);454 writel_relaxed(pmcfg1, pmu->base + PMCFG1);455 456 pmcfg2 = readl_relaxed(pmu->base + PMCFG2);457 pmcfg2 &= ~FIELD_PREP(MX93_PMCFG2_ID, 0x3FFFF);458 pmcfg2 |= FIELD_PREP(MX93_PMCFG2_ID, axi_id);459 writel_relaxed(pmcfg2, pmu->base + PMCFG2);460}461 462static void imx95_ddr_perf_monitor_config(struct ddr_pmu *pmu, int event,463 int counter, int axi_id, int axi_mask)464{465 u32 pmcfg1, pmcfg, offset = 0;466 467 pmcfg1 = readl_relaxed(pmu->base + PMCFG1);468 469 if (event == 73) {470 switch (counter) {471 case 2:472 pmcfg1 |= MX95_PMCFG1_WR_BEAT_FILT_EN;473 offset = PMCFG3;474 break;475 case 3:476 pmcfg1 |= MX95_PMCFG1_RD_BEAT_FILT_EN;477 offset = PMCFG4;478 break;479 case 4:480 pmcfg1 |= MX95_PMCFG1_RD_BEAT_FILT_EN;481 offset = PMCFG5;482 break;483 case 5:484 pmcfg1 |= MX95_PMCFG1_RD_BEAT_FILT_EN;485 offset = PMCFG6;486 break;487 }488 } else {489 switch (counter) {490 case 2:491 pmcfg1 &= ~MX95_PMCFG1_WR_BEAT_FILT_EN;492 break;493 case 3:494 case 4:495 case 5:496 pmcfg1 &= ~MX95_PMCFG1_RD_BEAT_FILT_EN;497 break;498 }499 }500 501 writel_relaxed(pmcfg1, pmu->base + PMCFG1);502 503 if (offset) {504 pmcfg = readl_relaxed(pmu->base + offset);505 pmcfg &= ~(FIELD_PREP(MX95_PMCFG_ID_MASK, 0x3FF) |506 FIELD_PREP(MX95_PMCFG_ID, 0x3FF));507 pmcfg |= (FIELD_PREP(MX95_PMCFG_ID_MASK, axi_mask) |508 FIELD_PREP(MX95_PMCFG_ID, axi_id));509 writel_relaxed(pmcfg, pmu->base + offset);510 }511}512 513static void ddr_perf_event_update(struct perf_event *event)514{515 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);516 struct hw_perf_event *hwc = &event->hw;517 int counter = hwc->idx;518 u64 new_raw_count;519 520 new_raw_count = ddr_perf_read_counter(pmu, counter);521 local64_add(new_raw_count, &event->count);522 523 /* clear counter's value every time */524 ddr_perf_clear_counter(pmu, counter);525}526 527static int ddr_perf_event_init(struct perf_event *event)528{529 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);530 struct hw_perf_event *hwc = &event->hw;531 struct perf_event *sibling;532 533 if (event->attr.type != event->pmu->type)534 return -ENOENT;535 536 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)537 return -EOPNOTSUPP;538 539 if (event->cpu < 0) {540 dev_warn(pmu->dev, "Can't provide per-task data!\n");541 return -EOPNOTSUPP;542 }543 544 /*545 * We must NOT create groups containing mixed PMUs, although software546 * events are acceptable (for example to create a CCN group547 * periodically read when a hrtimer aka cpu-clock leader triggers).548 */549 if (event->group_leader->pmu != event->pmu &&550 !is_software_event(event->group_leader))551 return -EINVAL;552 553 for_each_sibling_event(sibling, event->group_leader) {554 if (sibling->pmu != event->pmu &&555 !is_software_event(sibling))556 return -EINVAL;557 }558 559 event->cpu = pmu->cpu;560 hwc->idx = -1;561 562 return 0;563}564 565static void ddr_perf_event_start(struct perf_event *event, int flags)566{567 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);568 struct hw_perf_event *hwc = &event->hw;569 int counter = hwc->idx;570 571 local64_set(&hwc->prev_count, 0);572 573 ddr_perf_counter_local_config(pmu, event->attr.config, counter, true);574 hwc->state = 0;575}576 577static int ddr_perf_alloc_counter(struct ddr_pmu *pmu, int event, int counter)578{579 int i;580 581 if (event == CYCLES_EVENT_ID) {582 // Cycles counter is dedicated for cycle event.583 if (pmu->events[CYCLES_COUNTER] == NULL)584 return CYCLES_COUNTER;585 } else if (counter != 0) {586 // Counter specific event use specific counter.587 if (pmu->events[counter] == NULL)588 return counter;589 } else {590 // Auto allocate counter for referene event.591 for (i = 1; i < NUM_COUNTERS; i++)592 if (pmu->events[i] == NULL)593 return i;594 }595 596 return -ENOENT;597}598 599static int ddr_perf_event_add(struct perf_event *event, int flags)600{601 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);602 struct hw_perf_event *hwc = &event->hw;603 int cfg = event->attr.config;604 int cfg1 = event->attr.config1;605 int cfg2 = event->attr.config2;606 int event_id, counter;607 608 event_id = FIELD_GET(CONFIG_EVENT_MASK, cfg);609 counter = FIELD_GET(CONFIG_COUNTER_MASK, cfg);610 611 counter = ddr_perf_alloc_counter(pmu, event_id, counter);612 if (counter < 0) {613 dev_dbg(pmu->dev, "There are not enough counters\n");614 return -EOPNOTSUPP;615 }616 617 pmu->events[counter] = event;618 pmu->active_events++;619 hwc->idx = counter;620 hwc->state |= PERF_HES_STOPPED;621 622 if (is_imx93(pmu))623 /* read trans, write trans, read beat */624 imx93_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);625 626 if (is_imx95(pmu))627 /* write beat, read beat2, read beat1, read beat */628 imx95_ddr_perf_monitor_config(pmu, event_id, counter, cfg1, cfg2);629 630 if (flags & PERF_EF_START)631 ddr_perf_event_start(event, flags);632 633 return 0;634}635 636static void ddr_perf_event_stop(struct perf_event *event, int flags)637{638 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);639 struct hw_perf_event *hwc = &event->hw;640 int counter = hwc->idx;641 642 ddr_perf_counter_local_config(pmu, event->attr.config, counter, false);643 ddr_perf_event_update(event);644 645 hwc->state |= PERF_HES_STOPPED;646}647 648static void ddr_perf_event_del(struct perf_event *event, int flags)649{650 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);651 struct hw_perf_event *hwc = &event->hw;652 int counter = hwc->idx;653 654 ddr_perf_event_stop(event, PERF_EF_UPDATE);655 656 pmu->events[counter] = NULL;657 pmu->active_events--;658 hwc->idx = -1;659}660 661static void ddr_perf_pmu_enable(struct pmu *pmu)662{663 struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);664 665 ddr_perf_counter_global_config(ddr_pmu, true);666}667 668static void ddr_perf_pmu_disable(struct pmu *pmu)669{670 struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);671 672 ddr_perf_counter_global_config(ddr_pmu, false);673}674 675static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,676 struct device *dev)677{678 *pmu = (struct ddr_pmu) {679 .pmu = (struct pmu) {680 .module = THIS_MODULE,681 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,682 .task_ctx_nr = perf_invalid_context,683 .attr_groups = attr_groups,684 .event_init = ddr_perf_event_init,685 .add = ddr_perf_event_add,686 .del = ddr_perf_event_del,687 .start = ddr_perf_event_start,688 .stop = ddr_perf_event_stop,689 .read = ddr_perf_event_update,690 .pmu_enable = ddr_perf_pmu_enable,691 .pmu_disable = ddr_perf_pmu_disable,692 },693 .base = base,694 .dev = dev,695 };696}697 698static irqreturn_t ddr_perf_irq_handler(int irq, void *p)699{700 struct ddr_pmu *pmu = (struct ddr_pmu *)p;701 struct perf_event *event;702 int i;703 704 /*705 * Counters can generate an interrupt on an overflow when msb of a706 * counter changes from 0 to 1. For the interrupt to be signalled,707 * below condition mush be satisfied:708 * PMGC0[PMIE] = 1, PMGC0[FCECE] = 1, PMLCAn[CE] = 1709 * When an interrupt is signalled, PMGC0[FAC] is set by hardware and710 * all of the registers are frozen.711 * Software can clear the interrupt condition by resetting the performance712 * monitor and clearing the most significant bit of the counter that713 * generate the overflow.714 */715 for (i = 0; i < NUM_COUNTERS; i++) {716 if (!pmu->events[i])717 continue;718 719 event = pmu->events[i];720 721 ddr_perf_event_update(event);722 }723 724 ddr_perf_counter_global_config(pmu, true);725 726 return IRQ_HANDLED;727}728 729static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)730{731 struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);732 int target;733 734 if (cpu != pmu->cpu)735 return 0;736 737 target = cpumask_any_but(cpu_online_mask, cpu);738 if (target >= nr_cpu_ids)739 return 0;740 741 perf_pmu_migrate_context(&pmu->pmu, cpu, target);742 pmu->cpu = target;743 744 WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));745 746 return 0;747}748 749static int ddr_perf_probe(struct platform_device *pdev)750{751 struct ddr_pmu *pmu;752 void __iomem *base;753 int ret, irq;754 char *name;755 756 base = devm_platform_ioremap_resource(pdev, 0);757 if (IS_ERR(base))758 return PTR_ERR(base);759 760 pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);761 if (!pmu)762 return -ENOMEM;763 764 ddr_perf_init(pmu, base, &pdev->dev);765 766 pmu->devtype_data = of_device_get_match_data(&pdev->dev);767 768 platform_set_drvdata(pdev, pmu);769 770 pmu->id = ida_alloc(&ddr_ida, GFP_KERNEL);771 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", pmu->id);772 if (!name) {773 ret = -ENOMEM;774 goto format_string_err;775 }776 777 pmu->cpu = raw_smp_processor_id();778 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DDR_CPUHP_CB_NAME,779 NULL, ddr_perf_offline_cpu);780 if (ret < 0) {781 dev_err(&pdev->dev, "Failed to add callbacks for multi state\n");782 goto cpuhp_state_err;783 }784 pmu->cpuhp_state = ret;785 786 /* Register the pmu instance for cpu hotplug */787 ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);788 if (ret) {789 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);790 goto cpuhp_instance_err;791 }792 793 /* Request irq */794 irq = platform_get_irq(pdev, 0);795 if (irq < 0) {796 ret = irq;797 goto ddr_perf_err;798 }799 800 ret = devm_request_irq(&pdev->dev, irq, ddr_perf_irq_handler,801 IRQF_NOBALANCING | IRQF_NO_THREAD,802 DDR_CPUHP_CB_NAME, pmu);803 if (ret < 0) {804 dev_err(&pdev->dev, "Request irq failed: %d", ret);805 goto ddr_perf_err;806 }807 808 pmu->irq = irq;809 ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));810 if (ret) {811 dev_err(pmu->dev, "Failed to set interrupt affinity\n");812 goto ddr_perf_err;813 }814 815 ret = perf_pmu_register(&pmu->pmu, name, -1);816 if (ret)817 goto ddr_perf_err;818 819 return 0;820 821ddr_perf_err:822 cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);823cpuhp_instance_err:824 cpuhp_remove_multi_state(pmu->cpuhp_state);825cpuhp_state_err:826format_string_err:827 ida_free(&ddr_ida, pmu->id);828 dev_warn(&pdev->dev, "i.MX9 DDR Perf PMU failed (%d), disabled\n", ret);829 return ret;830}831 832static void ddr_perf_remove(struct platform_device *pdev)833{834 struct ddr_pmu *pmu = platform_get_drvdata(pdev);835 836 cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);837 cpuhp_remove_multi_state(pmu->cpuhp_state);838 839 perf_pmu_unregister(&pmu->pmu);840 841 ida_free(&ddr_ida, pmu->id);842}843 844static struct platform_driver imx_ddr_pmu_driver = {845 .driver = {846 .name = "imx9-ddr-pmu",847 .of_match_table = imx_ddr_pmu_dt_ids,848 .suppress_bind_attrs = true,849 },850 .probe = ddr_perf_probe,851 .remove_new = ddr_perf_remove,852};853module_platform_driver(imx_ddr_pmu_driver);854 855MODULE_AUTHOR("Xu Yang <xu.yang_2@nxp.com>");856MODULE_LICENSE("GPL v2");857MODULE_DESCRIPTION("DDRC PerfMon for i.MX9 SoCs");858