643 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * StarFive's StarLink PMU driver4 *5 * Copyright (C) 2023 StarFive Technology Co., Ltd.6 *7 * Author: Ji Sheng Teoh <jisheng.teoh@starfivetech.com>8 *9 */10 11#define STARLINK_PMU_PDEV_NAME "starfive_starlink_pmu"12#define pr_fmt(fmt) STARLINK_PMU_PDEV_NAME ": " fmt13 14#include <linux/bitmap.h>15#include <linux/cpu_pm.h>16#include <linux/io.h>17#include <linux/irq.h>18#include <linux/kernel.h>19#include <linux/module.h>20#include <linux/mod_devicetable.h>21#include <linux/perf_event.h>22#include <linux/platform_device.h>23#include <linux/sysfs.h>24 25#define STARLINK_PMU_MAX_COUNTERS 6426#define STARLINK_PMU_NUM_COUNTERS 1627#define STARLINK_PMU_IDX_CYCLE_COUNTER 6328 29#define STARLINK_PMU_EVENT_SELECT 0x06030#define STARLINK_PMU_EVENT_COUNTER 0x16031#define STARLINK_PMU_COUNTER_MASK GENMASK_ULL(63, 0)32#define STARLINK_PMU_CYCLE_COUNTER 0x05833 34#define STARLINK_PMU_CONTROL 0x04035#define STARLINK_PMU_GLOBAL_ENABLE BIT_ULL(0)36 37#define STARLINK_PMU_INTERRUPT_ENABLE 0x05038#define STARLINK_PMU_COUNTER_OVERFLOW_STATUS 0x04839#define STARLINK_PMU_CYCLE_OVERFLOW_MASK BIT_ULL(63)40 41#define STARLINK_CYCLES 0x05842#define CACHE_READ_REQUEST 0x0400070143#define CACHE_WRITE_REQUEST 0x0300000144#define CACHE_RELEASE_REQUEST 0x0003e00145#define CACHE_READ_HIT 0x0090120246#define CACHE_READ_MISS 0x0400800247#define CACHE_WRITE_HIT 0x006c000248#define CACHE_WRITE_MISS 0x0300000249#define CACHE_WRITEBACK 0x0000040350 51#define to_starlink_pmu(p) (container_of(p, struct starlink_pmu, pmu))52 53#define STARLINK_FORMAT_ATTR(_name, _config) \54 (&((struct dev_ext_attribute[]) { \55 { .attr = __ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \56 .var = (void *)_config, } \57 })[0].attr.attr)58 59#define STARLINK_EVENT_ATTR(_name, _id) \60 PMU_EVENT_ATTR_ID(_name, starlink_pmu_sysfs_event_show, _id)61 62static int starlink_pmu_cpuhp_state;63 64struct starlink_hw_events {65 struct perf_event *events[STARLINK_PMU_MAX_COUNTERS];66 DECLARE_BITMAP(used_mask, STARLINK_PMU_MAX_COUNTERS);67};68 69struct starlink_pmu {70 struct pmu pmu;71 struct starlink_hw_events __percpu *hw_events;72 struct hlist_node node;73 struct notifier_block starlink_pmu_pm_nb;74 void __iomem *pmu_base;75 cpumask_t cpumask;76 int irq;77};78 79static ssize_t80starlink_pmu_sysfs_format_show(struct device *dev,81 struct device_attribute *attr,82 char *buf)83{84 struct dev_ext_attribute *eattr = container_of(attr,85 struct dev_ext_attribute, attr);86 87 return sysfs_emit(buf, "%s\n", (char *)eattr->var);88}89 90static struct attribute *starlink_pmu_format_attrs[] = {91 STARLINK_FORMAT_ATTR(event, "config:0-31"),92 NULL93};94 95static const struct attribute_group starlink_pmu_format_attr_group = {96 .name = "format",97 .attrs = starlink_pmu_format_attrs,98};99 100static ssize_t101starlink_pmu_sysfs_event_show(struct device *dev,102 struct device_attribute *attr,103 char *buf)104{105 struct perf_pmu_events_attr *eattr = container_of(attr,106 struct perf_pmu_events_attr, attr);107 108 return sysfs_emit(buf, "event=0x%02llx\n", eattr->id);109}110 111static struct attribute *starlink_pmu_event_attrs[] = {112 STARLINK_EVENT_ATTR(cycles, STARLINK_CYCLES),113 STARLINK_EVENT_ATTR(read_request, CACHE_READ_REQUEST),114 STARLINK_EVENT_ATTR(write_request, CACHE_WRITE_REQUEST),115 STARLINK_EVENT_ATTR(release_request, CACHE_RELEASE_REQUEST),116 STARLINK_EVENT_ATTR(read_hit, CACHE_READ_HIT),117 STARLINK_EVENT_ATTR(read_miss, CACHE_READ_MISS),118 STARLINK_EVENT_ATTR(write_hit, CACHE_WRITE_HIT),119 STARLINK_EVENT_ATTR(write_miss, CACHE_WRITE_MISS),120 STARLINK_EVENT_ATTR(writeback, CACHE_WRITEBACK),121 NULL122};123 124static const struct attribute_group starlink_pmu_events_attr_group = {125 .name = "events",126 .attrs = starlink_pmu_event_attrs,127};128 129static ssize_t130cpumask_show(struct device *dev, struct device_attribute *attr, char *buf)131{132 struct starlink_pmu *starlink_pmu = to_starlink_pmu(dev_get_drvdata(dev));133 134 return cpumap_print_to_pagebuf(true, buf, &starlink_pmu->cpumask);135}136 137static DEVICE_ATTR_RO(cpumask);138 139static struct attribute *starlink_pmu_cpumask_attrs[] = {140 &dev_attr_cpumask.attr,141 NULL142};143 144static const struct attribute_group starlink_pmu_cpumask_attr_group = {145 .attrs = starlink_pmu_cpumask_attrs,146};147 148static const struct attribute_group *starlink_pmu_attr_groups[] = {149 &starlink_pmu_format_attr_group,150 &starlink_pmu_events_attr_group,151 &starlink_pmu_cpumask_attr_group,152 NULL153};154 155static void starlink_pmu_set_event_period(struct perf_event *event)156{157 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);158 struct hw_perf_event *hwc = &event->hw;159 int idx = event->hw.idx;160 161 /*162 * Program counter to half of it's max count to handle163 * cases of extreme interrupt latency.164 */165 u64 val = STARLINK_PMU_COUNTER_MASK >> 1;166 167 local64_set(&hwc->prev_count, val);168 if (hwc->config == STARLINK_CYCLES)169 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_CYCLE_COUNTER);170 else171 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_EVENT_COUNTER +172 idx * sizeof(u64));173}174 175static void starlink_pmu_counter_start(struct perf_event *event,176 struct starlink_pmu *starlink_pmu)177{178 struct hw_perf_event *hwc = &event->hw;179 int idx = event->hw.idx;180 u64 val;181 182 /*183 * Enable counter overflow interrupt[63:0],184 * which is mapped as follow:185 *186 * event counter 0 - Bit [0]187 * event counter 1 - Bit [1]188 * ...189 * cycle counter - Bit [63]190 */191 val = readq(starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);192 193 if (hwc->config == STARLINK_CYCLES) {194 /*195 * Cycle count has its dedicated register, and it starts196 * counting as soon as STARLINK_PMU_GLOBAL_ENABLE is set.197 */198 val |= STARLINK_PMU_CYCLE_OVERFLOW_MASK;199 } else {200 writeq(event->hw.config, starlink_pmu->pmu_base +201 STARLINK_PMU_EVENT_SELECT + idx * sizeof(u64));202 203 val |= BIT_ULL(idx);204 }205 206 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);207 208 writeq(STARLINK_PMU_GLOBAL_ENABLE, starlink_pmu->pmu_base +209 STARLINK_PMU_CONTROL);210}211 212static void starlink_pmu_counter_stop(struct perf_event *event,213 struct starlink_pmu *starlink_pmu)214{215 struct hw_perf_event *hwc = &event->hw;216 int idx = event->hw.idx;217 u64 val;218 219 val = readq(starlink_pmu->pmu_base + STARLINK_PMU_CONTROL);220 val &= ~STARLINK_PMU_GLOBAL_ENABLE;221 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_CONTROL);222 223 val = readq(starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);224 if (hwc->config == STARLINK_CYCLES)225 val &= ~STARLINK_PMU_CYCLE_OVERFLOW_MASK;226 else227 val &= ~BIT_ULL(idx);228 229 writeq(val, starlink_pmu->pmu_base + STARLINK_PMU_INTERRUPT_ENABLE);230}231 232static void starlink_pmu_update(struct perf_event *event)233{234 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);235 struct hw_perf_event *hwc = &event->hw;236 int idx = hwc->idx;237 u64 prev_raw_count, new_raw_count;238 u64 oldval;239 u64 delta;240 241 do {242 prev_raw_count = local64_read(&hwc->prev_count);243 if (hwc->config == STARLINK_CYCLES)244 new_raw_count = readq(starlink_pmu->pmu_base +245 STARLINK_PMU_CYCLE_COUNTER);246 else247 new_raw_count = readq(starlink_pmu->pmu_base +248 STARLINK_PMU_EVENT_COUNTER +249 idx * sizeof(u64));250 oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count,251 new_raw_count);252 } while (oldval != prev_raw_count);253 254 delta = (new_raw_count - prev_raw_count) & STARLINK_PMU_COUNTER_MASK;255 local64_add(delta, &event->count);256}257 258static void starlink_pmu_start(struct perf_event *event, int flags)259{260 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);261 struct hw_perf_event *hwc = &event->hw;262 263 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))264 return;265 266 if (flags & PERF_EF_RELOAD)267 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));268 269 hwc->state = 0;270 271 starlink_pmu_set_event_period(event);272 starlink_pmu_counter_start(event, starlink_pmu);273 274 perf_event_update_userpage(event);275}276 277static void starlink_pmu_stop(struct perf_event *event, int flags)278{279 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);280 struct hw_perf_event *hwc = &event->hw;281 282 if (hwc->state & PERF_HES_STOPPED)283 return;284 285 starlink_pmu_counter_stop(event, starlink_pmu);286 starlink_pmu_update(event);287 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;288}289 290static int starlink_pmu_add(struct perf_event *event, int flags)291{292 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);293 struct starlink_hw_events *hw_events =294 this_cpu_ptr(starlink_pmu->hw_events);295 struct hw_perf_event *hwc = &event->hw;296 unsigned long *used_mask = hw_events->used_mask;297 u32 n_events = STARLINK_PMU_NUM_COUNTERS;298 int idx;299 300 /*301 * Cycle counter has dedicated register to hold counter value.302 * Event other than cycle count has to be enabled through303 * event select register, and assigned with independent counter304 * as they appear.305 */306 307 if (hwc->config == STARLINK_CYCLES) {308 idx = STARLINK_PMU_IDX_CYCLE_COUNTER;309 } else {310 idx = find_first_zero_bit(used_mask, n_events);311 /* All counter are in use */312 if (idx < 0)313 return idx;314 315 set_bit(idx, used_mask);316 }317 318 hwc->idx = idx;319 hw_events->events[idx] = event;320 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;321 322 if (flags & PERF_EF_START)323 starlink_pmu_start(event, PERF_EF_RELOAD);324 325 perf_event_update_userpage(event);326 327 return 0;328}329 330static void starlink_pmu_del(struct perf_event *event, int flags)331{332 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);333 struct starlink_hw_events *hw_events =334 this_cpu_ptr(starlink_pmu->hw_events);335 struct hw_perf_event *hwc = &event->hw;336 337 starlink_pmu_stop(event, PERF_EF_UPDATE);338 hw_events->events[hwc->idx] = NULL;339 clear_bit(hwc->idx, hw_events->used_mask);340 341 perf_event_update_userpage(event);342}343 344static bool starlink_pmu_validate_event_group(struct perf_event *event)345{346 struct perf_event *leader = event->group_leader;347 struct perf_event *sibling;348 int counter = 1;349 350 /*351 * Ensure hardware events in the group are on the same PMU,352 * software events are acceptable.353 */354 if (event->group_leader->pmu != event->pmu &&355 !is_software_event(event->group_leader))356 return false;357 358 for_each_sibling_event(sibling, leader) {359 if (sibling->pmu != event->pmu && !is_software_event(sibling))360 return false;361 362 counter++;363 }364 365 return counter <= STARLINK_PMU_NUM_COUNTERS;366}367 368static int starlink_pmu_event_init(struct perf_event *event)369{370 struct starlink_pmu *starlink_pmu = to_starlink_pmu(event->pmu);371 struct hw_perf_event *hwc = &event->hw;372 373 /*374 * Sampling is not supported, as counters are shared375 * by all CPU.376 */377 if (hwc->sample_period)378 return -EOPNOTSUPP;379 380 /*381 * Per-task and attach to a task are not supported,382 * as uncore events are not specific to any CPU.383 */384 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)385 return -EOPNOTSUPP;386 387 if (!starlink_pmu_validate_event_group(event))388 return -EINVAL;389 390 hwc->idx = -1;391 hwc->config = event->attr.config;392 event->cpu = cpumask_first(&starlink_pmu->cpumask);393 394 return 0;395}396 397static irqreturn_t starlink_pmu_handle_irq(int irq_num, void *data)398{399 struct starlink_pmu *starlink_pmu = data;400 struct starlink_hw_events *hw_events =401 this_cpu_ptr(starlink_pmu->hw_events);402 bool handled = false;403 int idx;404 u64 overflow_status;405 406 for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) {407 struct perf_event *event = hw_events->events[idx];408 409 if (!event)410 continue;411 412 overflow_status = readq(starlink_pmu->pmu_base +413 STARLINK_PMU_COUNTER_OVERFLOW_STATUS);414 if (!(overflow_status & BIT_ULL(idx)))415 continue;416 417 writeq(BIT_ULL(idx), starlink_pmu->pmu_base +418 STARLINK_PMU_COUNTER_OVERFLOW_STATUS);419 420 starlink_pmu_update(event);421 starlink_pmu_set_event_period(event);422 handled = true;423 }424 return IRQ_RETVAL(handled);425}426 427static int starlink_setup_irqs(struct starlink_pmu *starlink_pmu,428 struct platform_device *pdev)429{430 int ret, irq;431 432 irq = platform_get_irq(pdev, 0);433 if (irq < 0)434 return -EINVAL;435 436 ret = devm_request_irq(&pdev->dev, irq, starlink_pmu_handle_irq,437 0, STARLINK_PMU_PDEV_NAME, starlink_pmu);438 if (ret)439 return dev_err_probe(&pdev->dev, ret, "Failed to request IRQ\n");440 441 starlink_pmu->irq = irq;442 443 return 0;444}445 446static int starlink_pmu_pm_notify(struct notifier_block *b,447 unsigned long cmd, void *v)448{449 struct starlink_pmu *starlink_pmu = container_of(b, struct starlink_pmu,450 starlink_pmu_pm_nb);451 struct starlink_hw_events *hw_events =452 this_cpu_ptr(starlink_pmu->hw_events);453 int enabled = bitmap_weight(hw_events->used_mask,454 STARLINK_PMU_MAX_COUNTERS);455 struct perf_event *event;456 int idx;457 458 if (!enabled)459 return NOTIFY_OK;460 461 for (idx = 0; idx < STARLINK_PMU_MAX_COUNTERS; idx++) {462 event = hw_events->events[idx];463 if (!event)464 continue;465 466 switch (cmd) {467 case CPU_PM_ENTER:468 /* Stop and update the counter */469 starlink_pmu_stop(event, PERF_EF_UPDATE);470 break;471 case CPU_PM_EXIT:472 case CPU_PM_ENTER_FAILED:473 /* Restore and enable the counter */474 starlink_pmu_start(event, PERF_EF_RELOAD);475 break;476 default:477 break;478 }479 }480 481 return NOTIFY_OK;482}483 484static int starlink_pmu_pm_register(struct starlink_pmu *starlink_pmu)485{486 if (!IS_ENABLED(CONFIG_CPU_PM))487 return 0;488 489 starlink_pmu->starlink_pmu_pm_nb.notifier_call = starlink_pmu_pm_notify;490 return cpu_pm_register_notifier(&starlink_pmu->starlink_pmu_pm_nb);491}492 493static void starlink_pmu_pm_unregister(struct starlink_pmu *starlink_pmu)494{495 if (!IS_ENABLED(CONFIG_CPU_PM))496 return;497 498 cpu_pm_unregister_notifier(&starlink_pmu->starlink_pmu_pm_nb);499}500 501static void starlink_pmu_destroy(struct starlink_pmu *starlink_pmu)502{503 starlink_pmu_pm_unregister(starlink_pmu);504 cpuhp_state_remove_instance(starlink_pmu_cpuhp_state,505 &starlink_pmu->node);506}507 508static int starlink_pmu_probe(struct platform_device *pdev)509{510 struct starlink_pmu *starlink_pmu;511 struct starlink_hw_events *hw_events;512 struct resource *res;513 int cpuid, i, ret;514 515 starlink_pmu = devm_kzalloc(&pdev->dev, sizeof(*starlink_pmu), GFP_KERNEL);516 if (!starlink_pmu)517 return -ENOMEM;518 519 starlink_pmu->pmu_base =520 devm_platform_get_and_ioremap_resource(pdev, 0, &res);521 if (IS_ERR(starlink_pmu->pmu_base))522 return PTR_ERR(starlink_pmu->pmu_base);523 524 starlink_pmu->hw_events = alloc_percpu_gfp(struct starlink_hw_events,525 GFP_KERNEL);526 if (!starlink_pmu->hw_events) {527 dev_err(&pdev->dev, "Failed to allocate per-cpu PMU data\n");528 return -ENOMEM;529 }530 531 for_each_possible_cpu(cpuid) {532 hw_events = per_cpu_ptr(starlink_pmu->hw_events, cpuid);533 for (i = 0; i < STARLINK_PMU_MAX_COUNTERS; i++)534 hw_events->events[i] = NULL;535 }536 537 ret = starlink_setup_irqs(starlink_pmu, pdev);538 if (ret)539 return ret;540 541 ret = cpuhp_state_add_instance(starlink_pmu_cpuhp_state,542 &starlink_pmu->node);543 if (ret) {544 dev_err(&pdev->dev, "Failed to register hotplug\n");545 return ret;546 }547 548 ret = starlink_pmu_pm_register(starlink_pmu);549 if (ret) {550 cpuhp_state_remove_instance(starlink_pmu_cpuhp_state,551 &starlink_pmu->node);552 return ret;553 }554 555 starlink_pmu->pmu = (struct pmu) {556 .task_ctx_nr = perf_invalid_context,557 .event_init = starlink_pmu_event_init,558 .add = starlink_pmu_add,559 .del = starlink_pmu_del,560 .start = starlink_pmu_start,561 .stop = starlink_pmu_stop,562 .read = starlink_pmu_update,563 .attr_groups = starlink_pmu_attr_groups,564 };565 566 ret = perf_pmu_register(&starlink_pmu->pmu, STARLINK_PMU_PDEV_NAME, -1);567 if (ret)568 starlink_pmu_destroy(starlink_pmu);569 570 return ret;571}572 573static const struct of_device_id starlink_pmu_of_match[] = {574 { .compatible = "starfive,jh8100-starlink-pmu" },575 {}576};577MODULE_DEVICE_TABLE(of, starlink_pmu_of_match);578 579static struct platform_driver starlink_pmu_driver = {580 .driver = {581 .name = STARLINK_PMU_PDEV_NAME,582 .of_match_table = starlink_pmu_of_match,583 .suppress_bind_attrs = true,584 },585 .probe = starlink_pmu_probe,586};587 588static int589starlink_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)590{591 struct starlink_pmu *starlink_pmu = hlist_entry_safe(node,592 struct starlink_pmu,593 node);594 595 if (cpumask_empty(&starlink_pmu->cpumask))596 cpumask_set_cpu(cpu, &starlink_pmu->cpumask);597 598 WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(cpu)));599 600 return 0;601}602 603static int604starlink_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)605{606 struct starlink_pmu *starlink_pmu = hlist_entry_safe(node,607 struct starlink_pmu,608 node);609 unsigned int target;610 611 if (!cpumask_test_and_clear_cpu(cpu, &starlink_pmu->cpumask))612 return 0;613 614 target = cpumask_any_but(cpu_online_mask, cpu);615 if (target >= nr_cpu_ids)616 return 0;617 618 perf_pmu_migrate_context(&starlink_pmu->pmu, cpu, target);619 620 cpumask_set_cpu(target, &starlink_pmu->cpumask);621 WARN_ON(irq_set_affinity(starlink_pmu->irq, cpumask_of(target)));622 623 return 0;624}625 626static int __init starlink_pmu_init(void)627{628 int ret;629 630 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,631 "soc/starfive/starlink_pmu:online",632 starlink_pmu_online_cpu,633 starlink_pmu_offline_cpu);634 if (ret < 0)635 return ret;636 637 starlink_pmu_cpuhp_state = ret;638 639 return platform_driver_register(&starlink_pmu_driver);640}641 642device_initcall(starlink_pmu_init);643