839 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd4 * Author: Lin Huang <hl@rock-chips.com>5 */6 7#include <linux/clk.h>8#include <linux/devfreq-event.h>9#include <linux/kernel.h>10#include <linux/err.h>11#include <linux/init.h>12#include <linux/io.h>13#include <linux/mfd/syscon.h>14#include <linux/module.h>15#include <linux/platform_device.h>16#include <linux/regmap.h>17#include <linux/slab.h>18#include <linux/list.h>19#include <linux/seqlock.h>20#include <linux/of.h>21#include <linux/of_device.h>22#include <linux/bitfield.h>23#include <linux/bits.h>24#include <linux/perf_event.h>25 26#include <soc/rockchip/rockchip_grf.h>27#include <soc/rockchip/rk3399_grf.h>28#include <soc/rockchip/rk3568_grf.h>29#include <soc/rockchip/rk3588_grf.h>30 31#define DMC_MAX_CHANNELS 432 33#define HIWORD_UPDATE(val, mask) ((val) | (mask) << 16)34 35/* DDRMON_CTRL */36#define DDRMON_CTRL 0x0437#define DDRMON_CTRL_DDR4 BIT(5)38#define DDRMON_CTRL_LPDDR4 BIT(4)39#define DDRMON_CTRL_HARDWARE_EN BIT(3)40#define DDRMON_CTRL_LPDDR23 BIT(2)41#define DDRMON_CTRL_SOFTWARE_EN BIT(1)42#define DDRMON_CTRL_TIMER_CNT_EN BIT(0)43#define DDRMON_CTRL_DDR_TYPE_MASK (DDRMON_CTRL_DDR4 | \44 DDRMON_CTRL_LPDDR4 | \45 DDRMON_CTRL_LPDDR23)46 47#define DDRMON_CH0_WR_NUM 0x2048#define DDRMON_CH0_RD_NUM 0x2449#define DDRMON_CH0_COUNT_NUM 0x2850#define DDRMON_CH0_DFI_ACCESS_NUM 0x2c51#define DDRMON_CH1_COUNT_NUM 0x3c52#define DDRMON_CH1_DFI_ACCESS_NUM 0x4053 54#define PERF_EVENT_CYCLES 0x055#define PERF_EVENT_READ_BYTES 0x156#define PERF_EVENT_WRITE_BYTES 0x257#define PERF_EVENT_READ_BYTES0 0x358#define PERF_EVENT_WRITE_BYTES0 0x459#define PERF_EVENT_READ_BYTES1 0x560#define PERF_EVENT_WRITE_BYTES1 0x661#define PERF_EVENT_READ_BYTES2 0x762#define PERF_EVENT_WRITE_BYTES2 0x863#define PERF_EVENT_READ_BYTES3 0x964#define PERF_EVENT_WRITE_BYTES3 0xa65#define PERF_EVENT_BYTES 0xb66#define PERF_ACCESS_TYPE_MAX 0xc67 68/**69 * struct dmc_count_channel - structure to hold counter values from the DDR controller70 * @access: Number of read and write accesses71 * @clock_cycles: DDR clock cycles72 * @read_access: number of read accesses73 * @write_access: number of write accesses74 */75struct dmc_count_channel {76 u64 access;77 u64 clock_cycles;78 u64 read_access;79 u64 write_access;80};81 82struct dmc_count {83 struct dmc_count_channel c[DMC_MAX_CHANNELS];84};85 86/*87 * The dfi controller can monitor DDR load. It has an upper and lower threshold88 * for the operating points. Whenever the usage leaves these bounds an event is89 * generated to indicate the DDR frequency should be changed.90 */91struct rockchip_dfi {92 struct devfreq_event_dev *edev;93 struct devfreq_event_desc desc;94 struct dmc_count last_event_count;95 96 struct dmc_count last_perf_count;97 struct dmc_count total_count;98 seqlock_t count_seqlock; /* protects last_perf_count and total_count */99 100 struct device *dev;101 void __iomem *regs;102 struct regmap *regmap_pmu;103 struct clk *clk;104 int usecount;105 struct mutex mutex;106 u32 ddr_type;107 unsigned int channel_mask;108 unsigned int max_channels;109 enum cpuhp_state cpuhp_state;110 struct hlist_node node;111 struct pmu pmu;112 struct hrtimer timer;113 unsigned int cpu;114 int active_events;115 int burst_len;116 int buswidth[DMC_MAX_CHANNELS];117 int ddrmon_stride;118 bool ddrmon_ctrl_single;119};120 121static int rockchip_dfi_enable(struct rockchip_dfi *dfi)122{123 void __iomem *dfi_regs = dfi->regs;124 int i, ret = 0;125 126 mutex_lock(&dfi->mutex);127 128 dfi->usecount++;129 if (dfi->usecount > 1)130 goto out;131 132 ret = clk_prepare_enable(dfi->clk);133 if (ret) {134 dev_err(&dfi->edev->dev, "failed to enable dfi clk: %d\n", ret);135 goto out;136 }137 138 for (i = 0; i < dfi->max_channels; i++) {139 u32 ctrl = 0;140 141 if (!(dfi->channel_mask & BIT(i)))142 continue;143 144 /* clear DDRMON_CTRL setting */145 writel_relaxed(HIWORD_UPDATE(0, DDRMON_CTRL_TIMER_CNT_EN |146 DDRMON_CTRL_SOFTWARE_EN | DDRMON_CTRL_HARDWARE_EN),147 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);148 149 /* set ddr type to dfi */150 switch (dfi->ddr_type) {151 case ROCKCHIP_DDRTYPE_LPDDR2:152 case ROCKCHIP_DDRTYPE_LPDDR3:153 ctrl = DDRMON_CTRL_LPDDR23;154 break;155 case ROCKCHIP_DDRTYPE_LPDDR4:156 case ROCKCHIP_DDRTYPE_LPDDR4X:157 ctrl = DDRMON_CTRL_LPDDR4;158 break;159 default:160 break;161 }162 163 writel_relaxed(HIWORD_UPDATE(ctrl, DDRMON_CTRL_DDR_TYPE_MASK),164 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);165 166 /* enable count, use software mode */167 writel_relaxed(HIWORD_UPDATE(DDRMON_CTRL_SOFTWARE_EN, DDRMON_CTRL_SOFTWARE_EN),168 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);169 170 if (dfi->ddrmon_ctrl_single)171 break;172 }173out:174 mutex_unlock(&dfi->mutex);175 176 return ret;177}178 179static void rockchip_dfi_disable(struct rockchip_dfi *dfi)180{181 void __iomem *dfi_regs = dfi->regs;182 int i;183 184 mutex_lock(&dfi->mutex);185 186 dfi->usecount--;187 188 WARN_ON_ONCE(dfi->usecount < 0);189 190 if (dfi->usecount > 0)191 goto out;192 193 for (i = 0; i < dfi->max_channels; i++) {194 if (!(dfi->channel_mask & BIT(i)))195 continue;196 197 writel_relaxed(HIWORD_UPDATE(0, DDRMON_CTRL_SOFTWARE_EN),198 dfi_regs + i * dfi->ddrmon_stride + DDRMON_CTRL);199 200 if (dfi->ddrmon_ctrl_single)201 break;202 }203 204 clk_disable_unprepare(dfi->clk);205out:206 mutex_unlock(&dfi->mutex);207}208 209static void rockchip_dfi_read_counters(struct rockchip_dfi *dfi, struct dmc_count *res)210{211 u32 i;212 void __iomem *dfi_regs = dfi->regs;213 214 for (i = 0; i < dfi->max_channels; i++) {215 if (!(dfi->channel_mask & BIT(i)))216 continue;217 res->c[i].read_access = readl_relaxed(dfi_regs +218 DDRMON_CH0_RD_NUM + i * dfi->ddrmon_stride);219 res->c[i].write_access = readl_relaxed(dfi_regs +220 DDRMON_CH0_WR_NUM + i * dfi->ddrmon_stride);221 res->c[i].access = readl_relaxed(dfi_regs +222 DDRMON_CH0_DFI_ACCESS_NUM + i * dfi->ddrmon_stride);223 res->c[i].clock_cycles = readl_relaxed(dfi_regs +224 DDRMON_CH0_COUNT_NUM + i * dfi->ddrmon_stride);225 }226}227 228static int rockchip_dfi_event_disable(struct devfreq_event_dev *edev)229{230 struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev);231 232 rockchip_dfi_disable(dfi);233 234 return 0;235}236 237static int rockchip_dfi_event_enable(struct devfreq_event_dev *edev)238{239 struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev);240 241 return rockchip_dfi_enable(dfi);242}243 244static int rockchip_dfi_set_event(struct devfreq_event_dev *edev)245{246 return 0;247}248 249static int rockchip_dfi_get_event(struct devfreq_event_dev *edev,250 struct devfreq_event_data *edata)251{252 struct rockchip_dfi *dfi = devfreq_event_get_drvdata(edev);253 struct dmc_count count;254 struct dmc_count *last = &dfi->last_event_count;255 u32 access = 0, clock_cycles = 0;256 int i;257 258 rockchip_dfi_read_counters(dfi, &count);259 260 /* We can only report one channel, so find the busiest one */261 for (i = 0; i < dfi->max_channels; i++) {262 u32 a, c;263 264 if (!(dfi->channel_mask & BIT(i)))265 continue;266 267 a = count.c[i].access - last->c[i].access;268 c = count.c[i].clock_cycles - last->c[i].clock_cycles;269 270 if (a > access) {271 access = a;272 clock_cycles = c;273 }274 }275 276 edata->load_count = access * 4;277 edata->total_count = clock_cycles;278 279 dfi->last_event_count = count;280 281 return 0;282}283 284static const struct devfreq_event_ops rockchip_dfi_ops = {285 .disable = rockchip_dfi_event_disable,286 .enable = rockchip_dfi_event_enable,287 .get_event = rockchip_dfi_get_event,288 .set_event = rockchip_dfi_set_event,289};290 291#ifdef CONFIG_PERF_EVENTS292 293static void rockchip_ddr_perf_counters_add(struct rockchip_dfi *dfi,294 const struct dmc_count *now,295 struct dmc_count *res)296{297 const struct dmc_count *last = &dfi->last_perf_count;298 int i;299 300 for (i = 0; i < dfi->max_channels; i++) {301 res->c[i].read_access = dfi->total_count.c[i].read_access +302 (u32)(now->c[i].read_access - last->c[i].read_access);303 res->c[i].write_access = dfi->total_count.c[i].write_access +304 (u32)(now->c[i].write_access - last->c[i].write_access);305 res->c[i].access = dfi->total_count.c[i].access +306 (u32)(now->c[i].access - last->c[i].access);307 res->c[i].clock_cycles = dfi->total_count.c[i].clock_cycles +308 (u32)(now->c[i].clock_cycles - last->c[i].clock_cycles);309 }310}311 312static ssize_t ddr_perf_cpumask_show(struct device *dev,313 struct device_attribute *attr, char *buf)314{315 struct pmu *pmu = dev_get_drvdata(dev);316 struct rockchip_dfi *dfi = container_of(pmu, struct rockchip_dfi, pmu);317 318 return cpumap_print_to_pagebuf(true, buf, cpumask_of(dfi->cpu));319}320 321static struct device_attribute ddr_perf_cpumask_attr =322 __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL);323 324static struct attribute *ddr_perf_cpumask_attrs[] = {325 &ddr_perf_cpumask_attr.attr,326 NULL,327};328 329static const struct attribute_group ddr_perf_cpumask_attr_group = {330 .attrs = ddr_perf_cpumask_attrs,331};332 333PMU_EVENT_ATTR_STRING(cycles, ddr_pmu_cycles, "event="__stringify(PERF_EVENT_CYCLES))334 335#define DFI_PMU_EVENT_ATTR(_name, _var, _str) \336 PMU_EVENT_ATTR_STRING(_name, _var, _str); \337 PMU_EVENT_ATTR_STRING(_name.unit, _var##_unit, "MB"); \338 PMU_EVENT_ATTR_STRING(_name.scale, _var##_scale, "9.536743164e-07")339 340DFI_PMU_EVENT_ATTR(read-bytes0, ddr_pmu_read_bytes0, "event="__stringify(PERF_EVENT_READ_BYTES0));341DFI_PMU_EVENT_ATTR(write-bytes0, ddr_pmu_write_bytes0, "event="__stringify(PERF_EVENT_WRITE_BYTES0));342 343DFI_PMU_EVENT_ATTR(read-bytes1, ddr_pmu_read_bytes1, "event="__stringify(PERF_EVENT_READ_BYTES1));344DFI_PMU_EVENT_ATTR(write-bytes1, ddr_pmu_write_bytes1, "event="__stringify(PERF_EVENT_WRITE_BYTES1));345 346DFI_PMU_EVENT_ATTR(read-bytes2, ddr_pmu_read_bytes2, "event="__stringify(PERF_EVENT_READ_BYTES2));347DFI_PMU_EVENT_ATTR(write-bytes2, ddr_pmu_write_bytes2, "event="__stringify(PERF_EVENT_WRITE_BYTES2));348 349DFI_PMU_EVENT_ATTR(read-bytes3, ddr_pmu_read_bytes3, "event="__stringify(PERF_EVENT_READ_BYTES3));350DFI_PMU_EVENT_ATTR(write-bytes3, ddr_pmu_write_bytes3, "event="__stringify(PERF_EVENT_WRITE_BYTES3));351 352DFI_PMU_EVENT_ATTR(read-bytes, ddr_pmu_read_bytes, "event="__stringify(PERF_EVENT_READ_BYTES));353DFI_PMU_EVENT_ATTR(write-bytes, ddr_pmu_write_bytes, "event="__stringify(PERF_EVENT_WRITE_BYTES));354 355DFI_PMU_EVENT_ATTR(bytes, ddr_pmu_bytes, "event="__stringify(PERF_EVENT_BYTES));356 357#define DFI_ATTR_MB(_name) \358 &_name.attr.attr, \359 &_name##_unit.attr.attr, \360 &_name##_scale.attr.attr361 362static struct attribute *ddr_perf_events_attrs[] = {363 &ddr_pmu_cycles.attr.attr,364 DFI_ATTR_MB(ddr_pmu_read_bytes),365 DFI_ATTR_MB(ddr_pmu_write_bytes),366 DFI_ATTR_MB(ddr_pmu_read_bytes0),367 DFI_ATTR_MB(ddr_pmu_write_bytes0),368 DFI_ATTR_MB(ddr_pmu_read_bytes1),369 DFI_ATTR_MB(ddr_pmu_write_bytes1),370 DFI_ATTR_MB(ddr_pmu_read_bytes2),371 DFI_ATTR_MB(ddr_pmu_write_bytes2),372 DFI_ATTR_MB(ddr_pmu_read_bytes3),373 DFI_ATTR_MB(ddr_pmu_write_bytes3),374 DFI_ATTR_MB(ddr_pmu_bytes),375 NULL,376};377 378static const struct attribute_group ddr_perf_events_attr_group = {379 .name = "events",380 .attrs = ddr_perf_events_attrs,381};382 383PMU_FORMAT_ATTR(event, "config:0-7");384 385static struct attribute *ddr_perf_format_attrs[] = {386 &format_attr_event.attr,387 NULL,388};389 390static const struct attribute_group ddr_perf_format_attr_group = {391 .name = "format",392 .attrs = ddr_perf_format_attrs,393};394 395static const struct attribute_group *attr_groups[] = {396 &ddr_perf_events_attr_group,397 &ddr_perf_cpumask_attr_group,398 &ddr_perf_format_attr_group,399 NULL,400};401 402static int rockchip_ddr_perf_event_init(struct perf_event *event)403{404 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);405 406 if (event->attr.type != event->pmu->type)407 return -ENOENT;408 409 if (event->attach_state & PERF_ATTACH_TASK)410 return -EINVAL;411 412 if (event->cpu < 0) {413 dev_warn(dfi->dev, "Can't provide per-task data!\n");414 return -EINVAL;415 }416 417 return 0;418}419 420static u64 rockchip_ddr_perf_event_get_count(struct perf_event *event)421{422 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);423 int blen = dfi->burst_len;424 struct dmc_count total, now;425 unsigned int seq;426 u64 count = 0;427 int i;428 429 rockchip_dfi_read_counters(dfi, &now);430 431 do {432 seq = read_seqbegin(&dfi->count_seqlock);433 rockchip_ddr_perf_counters_add(dfi, &now, &total);434 } while (read_seqretry(&dfi->count_seqlock, seq));435 436 switch (event->attr.config) {437 case PERF_EVENT_CYCLES:438 count = total.c[0].clock_cycles;439 break;440 case PERF_EVENT_READ_BYTES:441 for (i = 0; i < dfi->max_channels; i++)442 count += total.c[i].read_access * blen * dfi->buswidth[i];443 break;444 case PERF_EVENT_WRITE_BYTES:445 for (i = 0; i < dfi->max_channels; i++)446 count += total.c[i].write_access * blen * dfi->buswidth[i];447 break;448 case PERF_EVENT_READ_BYTES0:449 count = total.c[0].read_access * blen * dfi->buswidth[0];450 break;451 case PERF_EVENT_WRITE_BYTES0:452 count = total.c[0].write_access * blen * dfi->buswidth[0];453 break;454 case PERF_EVENT_READ_BYTES1:455 count = total.c[1].read_access * blen * dfi->buswidth[1];456 break;457 case PERF_EVENT_WRITE_BYTES1:458 count = total.c[1].write_access * blen * dfi->buswidth[1];459 break;460 case PERF_EVENT_READ_BYTES2:461 count = total.c[2].read_access * blen * dfi->buswidth[2];462 break;463 case PERF_EVENT_WRITE_BYTES2:464 count = total.c[2].write_access * blen * dfi->buswidth[2];465 break;466 case PERF_EVENT_READ_BYTES3:467 count = total.c[3].read_access * blen * dfi->buswidth[3];468 break;469 case PERF_EVENT_WRITE_BYTES3:470 count = total.c[3].write_access * blen * dfi->buswidth[3];471 break;472 case PERF_EVENT_BYTES:473 for (i = 0; i < dfi->max_channels; i++)474 count += total.c[i].access * blen * dfi->buswidth[i];475 break;476 }477 478 return count;479}480 481static void rockchip_ddr_perf_event_update(struct perf_event *event)482{483 u64 now;484 s64 prev;485 486 if (event->attr.config >= PERF_ACCESS_TYPE_MAX)487 return;488 489 now = rockchip_ddr_perf_event_get_count(event);490 prev = local64_xchg(&event->hw.prev_count, now);491 local64_add(now - prev, &event->count);492}493 494static void rockchip_ddr_perf_event_start(struct perf_event *event, int flags)495{496 u64 now = rockchip_ddr_perf_event_get_count(event);497 498 local64_set(&event->hw.prev_count, now);499}500 501static int rockchip_ddr_perf_event_add(struct perf_event *event, int flags)502{503 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);504 505 dfi->active_events++;506 507 if (dfi->active_events == 1) {508 dfi->total_count = (struct dmc_count){};509 rockchip_dfi_read_counters(dfi, &dfi->last_perf_count);510 hrtimer_start(&dfi->timer, ns_to_ktime(NSEC_PER_SEC), HRTIMER_MODE_REL);511 }512 513 if (flags & PERF_EF_START)514 rockchip_ddr_perf_event_start(event, flags);515 516 return 0;517}518 519static void rockchip_ddr_perf_event_stop(struct perf_event *event, int flags)520{521 rockchip_ddr_perf_event_update(event);522}523 524static void rockchip_ddr_perf_event_del(struct perf_event *event, int flags)525{526 struct rockchip_dfi *dfi = container_of(event->pmu, struct rockchip_dfi, pmu);527 528 rockchip_ddr_perf_event_stop(event, PERF_EF_UPDATE);529 530 dfi->active_events--;531 532 if (dfi->active_events == 0)533 hrtimer_cancel(&dfi->timer);534}535 536static enum hrtimer_restart rockchip_dfi_timer(struct hrtimer *timer)537{538 struct rockchip_dfi *dfi = container_of(timer, struct rockchip_dfi, timer);539 struct dmc_count now, total;540 541 rockchip_dfi_read_counters(dfi, &now);542 543 write_seqlock(&dfi->count_seqlock);544 545 rockchip_ddr_perf_counters_add(dfi, &now, &total);546 dfi->total_count = total;547 dfi->last_perf_count = now;548 549 write_sequnlock(&dfi->count_seqlock);550 551 hrtimer_forward_now(&dfi->timer, ns_to_ktime(NSEC_PER_SEC));552 553 return HRTIMER_RESTART;554};555 556static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)557{558 struct rockchip_dfi *dfi = hlist_entry_safe(node, struct rockchip_dfi, node);559 int target;560 561 if (cpu != dfi->cpu)562 return 0;563 564 target = cpumask_any_but(cpu_online_mask, cpu);565 if (target >= nr_cpu_ids)566 return 0;567 568 perf_pmu_migrate_context(&dfi->pmu, cpu, target);569 dfi->cpu = target;570 571 return 0;572}573 574static void rockchip_ddr_cpuhp_remove_state(void *data)575{576 struct rockchip_dfi *dfi = data;577 578 cpuhp_remove_multi_state(dfi->cpuhp_state);579 580 rockchip_dfi_disable(dfi);581}582 583static void rockchip_ddr_cpuhp_remove_instance(void *data)584{585 struct rockchip_dfi *dfi = data;586 587 cpuhp_state_remove_instance_nocalls(dfi->cpuhp_state, &dfi->node);588}589 590static void rockchip_ddr_perf_remove(void *data)591{592 struct rockchip_dfi *dfi = data;593 594 perf_pmu_unregister(&dfi->pmu);595}596 597static int rockchip_ddr_perf_init(struct rockchip_dfi *dfi)598{599 struct pmu *pmu = &dfi->pmu;600 int ret;601 602 seqlock_init(&dfi->count_seqlock);603 604 pmu->module = THIS_MODULE;605 pmu->capabilities = PERF_PMU_CAP_NO_EXCLUDE;606 pmu->task_ctx_nr = perf_invalid_context;607 pmu->attr_groups = attr_groups;608 pmu->event_init = rockchip_ddr_perf_event_init;609 pmu->add = rockchip_ddr_perf_event_add;610 pmu->del = rockchip_ddr_perf_event_del;611 pmu->start = rockchip_ddr_perf_event_start;612 pmu->stop = rockchip_ddr_perf_event_stop;613 pmu->read = rockchip_ddr_perf_event_update;614 615 dfi->cpu = raw_smp_processor_id();616 617 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,618 "rockchip_ddr_perf_pmu",619 NULL,620 ddr_perf_offline_cpu);621 622 if (ret < 0) {623 dev_err(dfi->dev, "cpuhp_setup_state_multi failed: %d\n", ret);624 return ret;625 }626 627 dfi->cpuhp_state = ret;628 629 rockchip_dfi_enable(dfi);630 631 ret = devm_add_action_or_reset(dfi->dev, rockchip_ddr_cpuhp_remove_state, dfi);632 if (ret)633 return ret;634 635 ret = cpuhp_state_add_instance_nocalls(dfi->cpuhp_state, &dfi->node);636 if (ret) {637 dev_err(dfi->dev, "Error %d registering hotplug\n", ret);638 return ret;639 }640 641 ret = devm_add_action_or_reset(dfi->dev, rockchip_ddr_cpuhp_remove_instance, dfi);642 if (ret)643 return ret;644 645 hrtimer_init(&dfi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);646 dfi->timer.function = rockchip_dfi_timer;647 648 switch (dfi->ddr_type) {649 case ROCKCHIP_DDRTYPE_LPDDR2:650 case ROCKCHIP_DDRTYPE_LPDDR3:651 dfi->burst_len = 8;652 break;653 case ROCKCHIP_DDRTYPE_LPDDR4:654 case ROCKCHIP_DDRTYPE_LPDDR4X:655 dfi->burst_len = 16;656 break;657 }658 659 ret = perf_pmu_register(pmu, "rockchip_ddr", -1);660 if (ret)661 return ret;662 663 return devm_add_action_or_reset(dfi->dev, rockchip_ddr_perf_remove, dfi);664}665#else666static int rockchip_ddr_perf_init(struct rockchip_dfi *dfi)667{668 return 0;669}670#endif671 672static int rk3399_dfi_init(struct rockchip_dfi *dfi)673{674 struct regmap *regmap_pmu = dfi->regmap_pmu;675 u32 val;676 677 dfi->clk = devm_clk_get(dfi->dev, "pclk_ddr_mon");678 if (IS_ERR(dfi->clk))679 return dev_err_probe(dfi->dev, PTR_ERR(dfi->clk),680 "Cannot get the clk pclk_ddr_mon\n");681 682 /* get ddr type */683 regmap_read(regmap_pmu, RK3399_PMUGRF_OS_REG2, &val);684 dfi->ddr_type = FIELD_GET(RK3399_PMUGRF_OS_REG2_DDRTYPE, val);685 686 dfi->channel_mask = GENMASK(1, 0);687 dfi->max_channels = 2;688 689 dfi->buswidth[0] = FIELD_GET(RK3399_PMUGRF_OS_REG2_BW_CH0, val) == 0 ? 4 : 2;690 dfi->buswidth[1] = FIELD_GET(RK3399_PMUGRF_OS_REG2_BW_CH1, val) == 0 ? 4 : 2;691 692 dfi->ddrmon_stride = 0x14;693 dfi->ddrmon_ctrl_single = true;694 695 return 0;696};697 698static int rk3568_dfi_init(struct rockchip_dfi *dfi)699{700 struct regmap *regmap_pmu = dfi->regmap_pmu;701 u32 reg2, reg3;702 703 regmap_read(regmap_pmu, RK3568_PMUGRF_OS_REG2, ®2);704 regmap_read(regmap_pmu, RK3568_PMUGRF_OS_REG3, ®3);705 706 /* lower 3 bits of the DDR type */707 dfi->ddr_type = FIELD_GET(RK3568_PMUGRF_OS_REG2_DRAMTYPE_INFO, reg2);708 709 /*710 * For version three and higher the upper two bits of the DDR type are711 * in RK3568_PMUGRF_OS_REG3712 */713 if (FIELD_GET(RK3568_PMUGRF_OS_REG3_SYSREG_VERSION, reg3) >= 0x3)714 dfi->ddr_type |= FIELD_GET(RK3568_PMUGRF_OS_REG3_DRAMTYPE_INFO_V3, reg3) << 3;715 716 dfi->channel_mask = BIT(0);717 dfi->max_channels = 1;718 719 dfi->buswidth[0] = FIELD_GET(RK3568_PMUGRF_OS_REG2_BW_CH0, reg2) == 0 ? 4 : 2;720 721 dfi->ddrmon_stride = 0x0; /* not relevant, we only have a single channel on this SoC */722 dfi->ddrmon_ctrl_single = true;723 724 return 0;725};726 727static int rk3588_dfi_init(struct rockchip_dfi *dfi)728{729 struct regmap *regmap_pmu = dfi->regmap_pmu;730 u32 reg2, reg3, reg4;731 732 regmap_read(regmap_pmu, RK3588_PMUGRF_OS_REG2, ®2);733 regmap_read(regmap_pmu, RK3588_PMUGRF_OS_REG3, ®3);734 regmap_read(regmap_pmu, RK3588_PMUGRF_OS_REG4, ®4);735 736 /* lower 3 bits of the DDR type */737 dfi->ddr_type = FIELD_GET(RK3588_PMUGRF_OS_REG2_DRAMTYPE_INFO, reg2);738 739 /*740 * For version three and higher the upper two bits of the DDR type are741 * in RK3588_PMUGRF_OS_REG3742 */743 if (FIELD_GET(RK3588_PMUGRF_OS_REG3_SYSREG_VERSION, reg3) >= 0x3)744 dfi->ddr_type |= FIELD_GET(RK3588_PMUGRF_OS_REG3_DRAMTYPE_INFO_V3, reg3) << 3;745 746 dfi->buswidth[0] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH0, reg2) == 0 ? 4 : 2;747 dfi->buswidth[1] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH1, reg2) == 0 ? 4 : 2;748 dfi->buswidth[2] = FIELD_GET(RK3568_PMUGRF_OS_REG2_BW_CH0, reg4) == 0 ? 4 : 2;749 dfi->buswidth[3] = FIELD_GET(RK3588_PMUGRF_OS_REG2_BW_CH1, reg4) == 0 ? 4 : 2;750 dfi->channel_mask = FIELD_GET(RK3588_PMUGRF_OS_REG2_CH_INFO, reg2) |751 FIELD_GET(RK3588_PMUGRF_OS_REG2_CH_INFO, reg4) << 2;752 dfi->max_channels = 4;753 754 dfi->ddrmon_stride = 0x4000;755 756 return 0;757};758 759static const struct of_device_id rockchip_dfi_id_match[] = {760 { .compatible = "rockchip,rk3399-dfi", .data = rk3399_dfi_init },761 { .compatible = "rockchip,rk3568-dfi", .data = rk3568_dfi_init },762 { .compatible = "rockchip,rk3588-dfi", .data = rk3588_dfi_init },763 { },764};765 766MODULE_DEVICE_TABLE(of, rockchip_dfi_id_match);767 768static int rockchip_dfi_probe(struct platform_device *pdev)769{770 struct device *dev = &pdev->dev;771 struct rockchip_dfi *dfi;772 struct devfreq_event_desc *desc;773 struct device_node *np = pdev->dev.of_node, *node;774 int (*soc_init)(struct rockchip_dfi *dfi);775 int ret;776 777 soc_init = of_device_get_match_data(&pdev->dev);778 if (!soc_init)779 return -EINVAL;780 781 dfi = devm_kzalloc(dev, sizeof(*dfi), GFP_KERNEL);782 if (!dfi)783 return -ENOMEM;784 785 dfi->regs = devm_platform_ioremap_resource(pdev, 0);786 if (IS_ERR(dfi->regs))787 return PTR_ERR(dfi->regs);788 789 node = of_parse_phandle(np, "rockchip,pmu", 0);790 if (!node)791 return dev_err_probe(&pdev->dev, -ENODEV, "Can't find pmu_grf registers\n");792 793 dfi->regmap_pmu = syscon_node_to_regmap(node);794 of_node_put(node);795 if (IS_ERR(dfi->regmap_pmu))796 return PTR_ERR(dfi->regmap_pmu);797 798 dfi->dev = dev;799 mutex_init(&dfi->mutex);800 801 desc = &dfi->desc;802 desc->ops = &rockchip_dfi_ops;803 desc->driver_data = dfi;804 desc->name = np->name;805 806 ret = soc_init(dfi);807 if (ret)808 return ret;809 810 dfi->edev = devm_devfreq_event_add_edev(&pdev->dev, desc);811 if (IS_ERR(dfi->edev)) {812 dev_err(&pdev->dev,813 "failed to add devfreq-event device\n");814 return PTR_ERR(dfi->edev);815 }816 817 ret = rockchip_ddr_perf_init(dfi);818 if (ret)819 return ret;820 821 platform_set_drvdata(pdev, dfi);822 823 return 0;824}825 826static struct platform_driver rockchip_dfi_driver = {827 .probe = rockchip_dfi_probe,828 .driver = {829 .name = "rockchip-dfi",830 .of_match_table = rockchip_dfi_id_match,831 .suppress_bind_attrs = true,832 },833};834module_platform_driver(rockchip_dfi_driver);835 836MODULE_LICENSE("GPL v2");837MODULE_AUTHOR("Lin Huang <hl@rock-chips.com>");838MODULE_DESCRIPTION("Rockchip DFI driver");839