768 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver3 *4 * Copyright (C) 2021 Marvell.5 */6 7#include <linux/init.h>8#include <linux/io.h>9#include <linux/module.h>10#include <linux/of.h>11#include <linux/perf_event.h>12#include <linux/hrtimer.h>13#include <linux/acpi.h>14#include <linux/platform_device.h>15 16/* Performance Counters Operating Mode Control Registers */17#define DDRC_PERF_CNT_OP_MODE_CTRL 0x802018#define OP_MODE_CTRL_VAL_MANNUAL 0x119 20/* Performance Counters Start Operation Control Registers */21#define DDRC_PERF_CNT_START_OP_CTRL 0x802822#define START_OP_CTRL_VAL_START 0x1ULL23#define START_OP_CTRL_VAL_ACTIVE 0x224 25/* Performance Counters End Operation Control Registers */26#define DDRC_PERF_CNT_END_OP_CTRL 0x803027#define END_OP_CTRL_VAL_END 0x1ULL28 29/* Performance Counters End Status Registers */30#define DDRC_PERF_CNT_END_STATUS 0x803831#define END_STATUS_VAL_END_TIMER_MODE_END 0x132 33/* Performance Counters Configuration Registers */34#define DDRC_PERF_CFG_BASE 0x804035 36/* 8 Generic event counter + 2 fixed event counters */37#define DDRC_PERF_NUM_GEN_COUNTERS 838#define DDRC_PERF_NUM_FIX_COUNTERS 239#define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS40#define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1)41#define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \42 DDRC_PERF_NUM_FIX_COUNTERS)43 44/* Generic event counter registers */45#define DDRC_PERF_CFG(n) (DDRC_PERF_CFG_BASE + 8 * (n))46#define EVENT_ENABLE BIT_ULL(63)47 48/* Two dedicated event counters for DDR reads and writes */49#define EVENT_DDR_READS 10150#define EVENT_DDR_WRITES 10051 52/*53 * programmable events IDs in programmable event counters.54 * DO NOT change these event-id numbers, they are used to55 * program event bitmap in h/w.56 */57#define EVENT_OP_IS_ZQLATCH 5558#define EVENT_OP_IS_ZQSTART 5459#define EVENT_OP_IS_TCR_MRR 5360#define EVENT_OP_IS_DQSOSC_MRR 5261#define EVENT_OP_IS_DQSOSC_MPC 5162#define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 5063#define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 4964#define EVENT_BSM_STARVATION 4865#define EVENT_BSM_ALLOC 4766#define EVENT_LPR_REQ_WITH_NOCREDIT 4667#define EVENT_HPR_REQ_WITH_NOCREDIT 4568#define EVENT_OP_IS_ZQCS 4469#define EVENT_OP_IS_ZQCL 4370#define EVENT_OP_IS_LOAD_MODE 4271#define EVENT_OP_IS_SPEC_REF 4172#define EVENT_OP_IS_CRIT_REF 4073#define EVENT_OP_IS_REFRESH 3974#define EVENT_OP_IS_ENTER_MPSM 3575#define EVENT_OP_IS_ENTER_POWERDOWN 3176#define EVENT_OP_IS_ENTER_SELFREF 2777#define EVENT_WAW_HAZARD 2678#define EVENT_RAW_HAZARD 2579#define EVENT_WAR_HAZARD 2480#define EVENT_WRITE_COMBINE 2381#define EVENT_RDWR_TRANSITIONS 2282#define EVENT_PRECHARGE_FOR_OTHER 2183#define EVENT_PRECHARGE_FOR_RDWR 2084#define EVENT_OP_IS_PRECHARGE 1985#define EVENT_OP_IS_MWR 1886#define EVENT_OP_IS_WR 1787#define EVENT_OP_IS_RD 1688#define EVENT_OP_IS_RD_ACTIVATE 1589#define EVENT_OP_IS_RD_OR_WR 1490#define EVENT_OP_IS_ACTIVATE 1391#define EVENT_WR_XACT_WHEN_CRITICAL 1292#define EVENT_LPR_XACT_WHEN_CRITICAL 1193#define EVENT_HPR_XACT_WHEN_CRITICAL 1094#define EVENT_DFI_RD_DATA_CYCLES 995#define EVENT_DFI_WR_DATA_CYCLES 896#define EVENT_ACT_BYPASS 797#define EVENT_READ_BYPASS 698#define EVENT_HIF_HI_PRI_RD 599#define EVENT_HIF_RMW 4100#define EVENT_HIF_RD 3101#define EVENT_HIF_WR 2102#define EVENT_HIF_RD_OR_WR 1103 104/* Event counter value registers */105#define DDRC_PERF_CNT_VALUE_BASE 0x8080106#define DDRC_PERF_CNT_VALUE(n) (DDRC_PERF_CNT_VALUE_BASE + 8 * (n))107 108/* Fixed event counter enable/disable register */109#define DDRC_PERF_CNT_FREERUN_EN 0x80C0110#define DDRC_PERF_FREERUN_WRITE_EN 0x1111#define DDRC_PERF_FREERUN_READ_EN 0x2112 113/* Fixed event counter control register */114#define DDRC_PERF_CNT_FREERUN_CTRL 0x80C8115#define DDRC_FREERUN_WRITE_CNT_CLR 0x1116#define DDRC_FREERUN_READ_CNT_CLR 0x2117 118/* Fixed event counter value register */119#define DDRC_PERF_CNT_VALUE_WR_OP 0x80D0120#define DDRC_PERF_CNT_VALUE_RD_OP 0x80D8121#define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48)122#define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0)123 124struct cn10k_ddr_pmu {125 struct pmu pmu;126 void __iomem *base;127 unsigned int cpu;128 struct device *dev;129 int active_events;130 struct perf_event *events[DDRC_PERF_NUM_COUNTERS];131 struct hrtimer hrtimer;132 struct hlist_node node;133};134 135#define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu)136 137static ssize_t cn10k_ddr_pmu_event_show(struct device *dev,138 struct device_attribute *attr,139 char *page)140{141 struct perf_pmu_events_attr *pmu_attr;142 143 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);144 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);145 146}147 148#define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \149 PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id)150 151static struct attribute *cn10k_ddr_perf_events_attrs[] = {152 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),153 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),154 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),155 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),156 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),157 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),158 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),159 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES),160 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES),161 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,162 EVENT_HPR_XACT_WHEN_CRITICAL),163 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,164 EVENT_LPR_XACT_WHEN_CRITICAL),165 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,166 EVENT_WR_XACT_WHEN_CRITICAL),167 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),168 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR),169 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE),170 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),171 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),172 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),173 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),174 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR),175 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,176 EVENT_PRECHARGE_FOR_OTHER),177 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),178 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),179 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),180 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),181 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),182 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),183 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN),184 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),185 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),186 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),187 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),188 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),189 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),190 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),191 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit,192 EVENT_HPR_REQ_WITH_NOCREDIT),193 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit,194 EVENT_LPR_REQ_WITH_NOCREDIT),195 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),196 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),197 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,198 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),199 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,200 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),201 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),202 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),203 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),204 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),205 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),206 /* Free run event counters */207 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),208 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),209 NULL210};211 212static struct attribute_group cn10k_ddr_perf_events_attr_group = {213 .name = "events",214 .attrs = cn10k_ddr_perf_events_attrs,215};216 217PMU_FORMAT_ATTR(event, "config:0-8");218 219static struct attribute *cn10k_ddr_perf_format_attrs[] = {220 &format_attr_event.attr,221 NULL,222};223 224static struct attribute_group cn10k_ddr_perf_format_attr_group = {225 .name = "format",226 .attrs = cn10k_ddr_perf_format_attrs,227};228 229static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,230 struct device_attribute *attr,231 char *buf)232{233 struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);234 235 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));236}237 238static struct device_attribute cn10k_ddr_perf_cpumask_attr =239 __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL);240 241static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = {242 &cn10k_ddr_perf_cpumask_attr.attr,243 NULL,244};245 246static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = {247 .attrs = cn10k_ddr_perf_cpumask_attrs,248};249 250static const struct attribute_group *cn10k_attr_groups[] = {251 &cn10k_ddr_perf_events_attr_group,252 &cn10k_ddr_perf_format_attr_group,253 &cn10k_ddr_perf_cpumask_attr_group,254 NULL,255};256 257/* Default poll timeout is 100 sec, which is very sufficient for258 * 48 bit counter incremented max at 5.6 GT/s, which may take many259 * hours to overflow.260 */261static unsigned long cn10k_ddr_pmu_poll_period_sec = 100;262module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644);263 264static ktime_t cn10k_ddr_pmu_timer_period(void)265{266 return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC);267}268 269static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap)270{271 switch (eventid) {272 case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD:273 case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH:274 *event_bitmap = (1ULL << (eventid - 1));275 break;276 case EVENT_OP_IS_ENTER_SELFREF:277 case EVENT_OP_IS_ENTER_POWERDOWN:278 case EVENT_OP_IS_ENTER_MPSM:279 *event_bitmap = (0xFULL << (eventid - 1));280 break;281 default:282 pr_err("%s Invalid eventid %d\n", __func__, eventid);283 return -EINVAL;284 }285 286 return 0;287}288 289static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu,290 struct perf_event *event)291{292 u8 config = event->attr.config;293 int i;294 295 /* DDR read free-run counter index */296 if (config == EVENT_DDR_READS) {297 pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event;298 return DDRC_PERF_READ_COUNTER_IDX;299 }300 301 /* DDR write free-run counter index */302 if (config == EVENT_DDR_WRITES) {303 pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event;304 return DDRC_PERF_WRITE_COUNTER_IDX;305 }306 307 /* Allocate DDR generic counters */308 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {309 if (pmu->events[i] == NULL) {310 pmu->events[i] = event;311 return i;312 }313 }314 315 return -ENOENT;316}317 318static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter)319{320 pmu->events[counter] = NULL;321}322 323static int cn10k_ddr_perf_event_init(struct perf_event *event)324{325 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);326 struct hw_perf_event *hwc = &event->hw;327 328 if (event->attr.type != event->pmu->type)329 return -ENOENT;330 331 if (is_sampling_event(event)) {332 dev_info(pmu->dev, "Sampling not supported!\n");333 return -EOPNOTSUPP;334 }335 336 if (event->cpu < 0) {337 dev_warn(pmu->dev, "Can't provide per-task data!\n");338 return -EOPNOTSUPP;339 }340 341 /* We must NOT create groups containing mixed PMUs */342 if (event->group_leader->pmu != event->pmu &&343 !is_software_event(event->group_leader))344 return -EINVAL;345 346 /* Set ownership of event to one CPU, same event can not be observed347 * on multiple cpus at same time.348 */349 event->cpu = pmu->cpu;350 hwc->idx = -1;351 return 0;352}353 354static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu,355 int counter, bool enable)356{357 u32 reg;358 u64 val;359 360 if (counter > DDRC_PERF_NUM_COUNTERS) {361 pr_err("Error: unsupported counter %d\n", counter);362 return;363 }364 365 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {366 reg = DDRC_PERF_CFG(counter);367 val = readq_relaxed(pmu->base + reg);368 369 if (enable)370 val |= EVENT_ENABLE;371 else372 val &= ~EVENT_ENABLE;373 374 writeq_relaxed(val, pmu->base + reg);375 } else {376 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_FREERUN_EN);377 if (enable) {378 if (counter == DDRC_PERF_READ_COUNTER_IDX)379 val |= DDRC_PERF_FREERUN_READ_EN;380 else381 val |= DDRC_PERF_FREERUN_WRITE_EN;382 } else {383 if (counter == DDRC_PERF_READ_COUNTER_IDX)384 val &= ~DDRC_PERF_FREERUN_READ_EN;385 else386 val &= ~DDRC_PERF_FREERUN_WRITE_EN;387 }388 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_EN);389 }390}391 392static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter)393{394 u64 val;395 396 if (counter == DDRC_PERF_READ_COUNTER_IDX)397 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_RD_OP);398 399 if (counter == DDRC_PERF_WRITE_COUNTER_IDX)400 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_WR_OP);401 402 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE(counter));403 return val;404}405 406static void cn10k_ddr_perf_event_update(struct perf_event *event)407{408 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);409 struct hw_perf_event *hwc = &event->hw;410 u64 prev_count, new_count, mask;411 412 do {413 prev_count = local64_read(&hwc->prev_count);414 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);415 } while (local64_xchg(&hwc->prev_count, new_count) != prev_count);416 417 mask = DDRC_PERF_CNT_MAX_VALUE;418 419 local64_add((new_count - prev_count) & mask, &event->count);420}421 422static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags)423{424 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);425 struct hw_perf_event *hwc = &event->hw;426 int counter = hwc->idx;427 428 local64_set(&hwc->prev_count, 0);429 430 cn10k_ddr_perf_counter_enable(pmu, counter, true);431 432 hwc->state = 0;433}434 435static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags)436{437 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);438 struct hw_perf_event *hwc = &event->hw;439 u8 config = event->attr.config;440 int counter, ret;441 u32 reg_offset;442 u64 val;443 444 counter = cn10k_ddr_perf_alloc_counter(pmu, event);445 if (counter < 0)446 return -EAGAIN;447 448 pmu->active_events++;449 hwc->idx = counter;450 451 if (pmu->active_events == 1)452 hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(),453 HRTIMER_MODE_REL_PINNED);454 455 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {456 /* Generic counters, configure event id */457 reg_offset = DDRC_PERF_CFG(counter);458 ret = ddr_perf_get_event_bitmap(config, &val);459 if (ret)460 return ret;461 462 writeq_relaxed(val, pmu->base + reg_offset);463 } else {464 /* fixed event counter, clear counter value */465 if (counter == DDRC_PERF_READ_COUNTER_IDX)466 val = DDRC_FREERUN_READ_CNT_CLR;467 else468 val = DDRC_FREERUN_WRITE_CNT_CLR;469 470 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_CTRL);471 }472 473 hwc->state |= PERF_HES_STOPPED;474 475 if (flags & PERF_EF_START)476 cn10k_ddr_perf_event_start(event, flags);477 478 return 0;479}480 481static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags)482{483 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);484 struct hw_perf_event *hwc = &event->hw;485 int counter = hwc->idx;486 487 cn10k_ddr_perf_counter_enable(pmu, counter, false);488 489 if (flags & PERF_EF_UPDATE)490 cn10k_ddr_perf_event_update(event);491 492 hwc->state |= PERF_HES_STOPPED;493}494 495static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags)496{497 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);498 struct hw_perf_event *hwc = &event->hw;499 int counter = hwc->idx;500 501 cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE);502 503 cn10k_ddr_perf_free_counter(pmu, counter);504 pmu->active_events--;505 hwc->idx = -1;506 507 /* Cancel timer when no events to capture */508 if (pmu->active_events == 0)509 hrtimer_cancel(&pmu->hrtimer);510}511 512static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu)513{514 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);515 516 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +517 DDRC_PERF_CNT_START_OP_CTRL);518}519 520static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu)521{522 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);523 524 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +525 DDRC_PERF_CNT_END_OP_CTRL);526}527 528static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu)529{530 struct hw_perf_event *hwc;531 int i;532 533 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {534 if (pmu->events[i] == NULL)535 continue;536 537 cn10k_ddr_perf_event_update(pmu->events[i]);538 }539 540 /* Reset previous count as h/w counter are reset */541 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {542 if (pmu->events[i] == NULL)543 continue;544 545 hwc = &pmu->events[i]->hw;546 local64_set(&hwc->prev_count, 0);547 }548}549 550static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu)551{552 struct perf_event *event;553 struct hw_perf_event *hwc;554 u64 prev_count, new_count;555 u64 value;556 int i;557 558 event = pmu->events[DDRC_PERF_READ_COUNTER_IDX];559 if (event) {560 hwc = &event->hw;561 prev_count = local64_read(&hwc->prev_count);562 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);563 564 /* Overflow condition is when new count less than565 * previous count566 */567 if (new_count < prev_count)568 cn10k_ddr_perf_event_update(event);569 }570 571 event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX];572 if (event) {573 hwc = &event->hw;574 prev_count = local64_read(&hwc->prev_count);575 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);576 577 /* Overflow condition is when new count less than578 * previous count579 */580 if (new_count < prev_count)581 cn10k_ddr_perf_event_update(event);582 }583 584 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {585 if (pmu->events[i] == NULL)586 continue;587 588 value = cn10k_ddr_perf_read_counter(pmu, i);589 if (value == DDRC_PERF_CNT_MAX_VALUE) {590 pr_info("Counter-(%d) reached max value\n", i);591 cn10k_ddr_perf_event_update_all(pmu);592 cn10k_ddr_perf_pmu_disable(&pmu->pmu);593 cn10k_ddr_perf_pmu_enable(&pmu->pmu);594 }595 }596 597 return IRQ_HANDLED;598}599 600static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer)601{602 struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu,603 hrtimer);604 unsigned long flags;605 606 local_irq_save(flags);607 cn10k_ddr_pmu_overflow_handler(pmu);608 local_irq_restore(flags);609 610 hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period());611 return HRTIMER_RESTART;612}613 614static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)615{616 struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu,617 node);618 unsigned int target;619 620 if (cpu != pmu->cpu)621 return 0;622 623 target = cpumask_any_but(cpu_online_mask, cpu);624 if (target >= nr_cpu_ids)625 return 0;626 627 perf_pmu_migrate_context(&pmu->pmu, cpu, target);628 pmu->cpu = target;629 return 0;630}631 632static int cn10k_ddr_perf_probe(struct platform_device *pdev)633{634 struct cn10k_ddr_pmu *ddr_pmu;635 struct resource *res;636 void __iomem *base;637 char *name;638 int ret;639 640 ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL);641 if (!ddr_pmu)642 return -ENOMEM;643 644 ddr_pmu->dev = &pdev->dev;645 platform_set_drvdata(pdev, ddr_pmu);646 647 base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);648 if (IS_ERR(base))649 return PTR_ERR(base);650 651 ddr_pmu->base = base;652 653 /* Setup the PMU counter to work in manual mode */654 writeq_relaxed(OP_MODE_CTRL_VAL_MANNUAL, ddr_pmu->base +655 DDRC_PERF_CNT_OP_MODE_CTRL);656 657 ddr_pmu->pmu = (struct pmu) {658 .module = THIS_MODULE,659 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,660 .task_ctx_nr = perf_invalid_context,661 .attr_groups = cn10k_attr_groups,662 .event_init = cn10k_ddr_perf_event_init,663 .add = cn10k_ddr_perf_event_add,664 .del = cn10k_ddr_perf_event_del,665 .start = cn10k_ddr_perf_event_start,666 .stop = cn10k_ddr_perf_event_stop,667 .read = cn10k_ddr_perf_event_update,668 .pmu_enable = cn10k_ddr_perf_pmu_enable,669 .pmu_disable = cn10k_ddr_perf_pmu_disable,670 };671 672 /* Choose this cpu to collect perf data */673 ddr_pmu->cpu = raw_smp_processor_id();674 675 name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx",676 res->start);677 if (!name)678 return -ENOMEM;679 680 hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);681 ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler;682 683 cpuhp_state_add_instance_nocalls(684 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,685 &ddr_pmu->node);686 687 ret = perf_pmu_register(&ddr_pmu->pmu, name, -1);688 if (ret)689 goto error;690 691 pr_info("CN10K DDR PMU Driver for ddrc@%llx\n", res->start);692 return 0;693error:694 cpuhp_state_remove_instance_nocalls(695 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,696 &ddr_pmu->node);697 return ret;698}699 700static void cn10k_ddr_perf_remove(struct platform_device *pdev)701{702 struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev);703 704 cpuhp_state_remove_instance_nocalls(705 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,706 &ddr_pmu->node);707 708 perf_pmu_unregister(&ddr_pmu->pmu);709}710 711#ifdef CONFIG_OF712static const struct of_device_id cn10k_ddr_pmu_of_match[] = {713 { .compatible = "marvell,cn10k-ddr-pmu", },714 { },715};716MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match);717#endif718 719#ifdef CONFIG_ACPI720static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = {721 {"MRVL000A", 0},722 {},723};724MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match);725#endif726 727static struct platform_driver cn10k_ddr_pmu_driver = {728 .driver = {729 .name = "cn10k-ddr-pmu",730 .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match),731 .acpi_match_table = ACPI_PTR(cn10k_ddr_pmu_acpi_match),732 .suppress_bind_attrs = true,733 },734 .probe = cn10k_ddr_perf_probe,735 .remove_new = cn10k_ddr_perf_remove,736};737 738static int __init cn10k_ddr_pmu_init(void)739{740 int ret;741 742 ret = cpuhp_setup_state_multi(743 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,744 "perf/marvell/cn10k/ddr:online", NULL,745 cn10k_ddr_pmu_offline_cpu);746 if (ret)747 return ret;748 749 ret = platform_driver_register(&cn10k_ddr_pmu_driver);750 if (ret)751 cpuhp_remove_multi_state(752 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);753 return ret;754}755 756static void __exit cn10k_ddr_pmu_exit(void)757{758 platform_driver_unregister(&cn10k_ddr_pmu_driver);759 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);760}761 762module_init(cn10k_ddr_pmu_init);763module_exit(cn10k_ddr_pmu_exit);764 765MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>");766MODULE_DESCRIPTION("Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver");767MODULE_LICENSE("GPL v2");768