542 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * HiSilicon SoC Hardware event counters support4 *5 * Copyright (C) 2017 HiSilicon Limited6 * Author: Anurup M <anurup.m@huawei.com>7 * Shaokun Zhang <zhangshaokun@hisilicon.com>8 *9 * This code is based on the uncore PMUs like arm-cci and arm-ccn.10 */11#include <linux/bitmap.h>12#include <linux/bitops.h>13#include <linux/bug.h>14#include <linux/err.h>15#include <linux/errno.h>16#include <linux/interrupt.h>17 18#include <asm/cputype.h>19#include <asm/local64.h>20 21#include "hisi_uncore_pmu.h"22 23#define HISI_MAX_PERIOD(nr) (GENMASK_ULL((nr) - 1, 0))24 25/*26 * PMU event attributes27 */28ssize_t hisi_event_sysfs_show(struct device *dev,29 struct device_attribute *attr, char *page)30{31 struct dev_ext_attribute *eattr;32 33 eattr = container_of(attr, struct dev_ext_attribute, attr);34 35 return sysfs_emit(page, "config=0x%lx\n", (unsigned long)eattr->var);36}37EXPORT_SYMBOL_GPL(hisi_event_sysfs_show);38 39/*40 * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show41 */42ssize_t hisi_cpumask_sysfs_show(struct device *dev,43 struct device_attribute *attr, char *buf)44{45 struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));46 47 return sysfs_emit(buf, "%d\n", hisi_pmu->on_cpu);48}49EXPORT_SYMBOL_GPL(hisi_cpumask_sysfs_show);50 51static bool hisi_validate_event_group(struct perf_event *event)52{53 struct perf_event *sibling, *leader = event->group_leader;54 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);55 /* Include count for the event */56 int counters = 1;57 58 if (!is_software_event(leader)) {59 /*60 * We must NOT create groups containing mixed PMUs, although61 * software events are acceptable62 */63 if (leader->pmu != event->pmu)64 return false;65 66 /* Increment counter for the leader */67 if (leader != event)68 counters++;69 }70 71 for_each_sibling_event(sibling, event->group_leader) {72 if (is_software_event(sibling))73 continue;74 if (sibling->pmu != event->pmu)75 return false;76 /* Increment counter for each sibling */77 counters++;78 }79 80 /* The group can not count events more than the counters in the HW */81 return counters <= hisi_pmu->num_counters;82}83 84int hisi_uncore_pmu_get_event_idx(struct perf_event *event)85{86 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);87 unsigned long *used_mask = hisi_pmu->pmu_events.used_mask;88 u32 num_counters = hisi_pmu->num_counters;89 int idx;90 91 idx = find_first_zero_bit(used_mask, num_counters);92 if (idx == num_counters)93 return -EAGAIN;94 95 set_bit(idx, used_mask);96 97 return idx;98}99EXPORT_SYMBOL_GPL(hisi_uncore_pmu_get_event_idx);100 101ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev,102 struct device_attribute *attr,103 char *page)104{105 struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));106 107 return sysfs_emit(page, "0x%08x\n", hisi_pmu->identifier);108}109EXPORT_SYMBOL_GPL(hisi_uncore_pmu_identifier_attr_show);110 111static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)112{113 clear_bit(idx, hisi_pmu->pmu_events.used_mask);114}115 116static irqreturn_t hisi_uncore_pmu_isr(int irq, void *data)117{118 struct hisi_pmu *hisi_pmu = data;119 struct perf_event *event;120 unsigned long overflown;121 int idx;122 123 overflown = hisi_pmu->ops->get_int_status(hisi_pmu);124 if (!overflown)125 return IRQ_NONE;126 127 /*128 * Find the counter index which overflowed if the bit was set129 * and handle it.130 */131 for_each_set_bit(idx, &overflown, hisi_pmu->num_counters) {132 /* Write 1 to clear the IRQ status flag */133 hisi_pmu->ops->clear_int_status(hisi_pmu, idx);134 /* Get the corresponding event struct */135 event = hisi_pmu->pmu_events.hw_events[idx];136 if (!event)137 continue;138 139 hisi_uncore_pmu_event_update(event);140 hisi_uncore_pmu_set_event_period(event);141 }142 143 return IRQ_HANDLED;144}145 146int hisi_uncore_pmu_init_irq(struct hisi_pmu *hisi_pmu,147 struct platform_device *pdev)148{149 int irq, ret;150 151 irq = platform_get_irq(pdev, 0);152 if (irq < 0)153 return irq;154 155 ret = devm_request_irq(&pdev->dev, irq, hisi_uncore_pmu_isr,156 IRQF_NOBALANCING | IRQF_NO_THREAD,157 dev_name(&pdev->dev), hisi_pmu);158 if (ret < 0) {159 dev_err(&pdev->dev,160 "Fail to request IRQ: %d ret: %d.\n", irq, ret);161 return ret;162 }163 164 hisi_pmu->irq = irq;165 166 return 0;167}168EXPORT_SYMBOL_GPL(hisi_uncore_pmu_init_irq);169 170int hisi_uncore_pmu_event_init(struct perf_event *event)171{172 struct hw_perf_event *hwc = &event->hw;173 struct hisi_pmu *hisi_pmu;174 175 if (event->attr.type != event->pmu->type)176 return -ENOENT;177 178 /*179 * We do not support sampling as the counters are all180 * shared by all CPU cores in a CPU die(SCCL). Also we181 * do not support attach to a task(per-process mode)182 */183 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)184 return -EOPNOTSUPP;185 186 /*187 * The uncore counters not specific to any CPU, so cannot188 * support per-task189 */190 if (event->cpu < 0)191 return -EINVAL;192 193 /*194 * Validate if the events in group does not exceed the195 * available counters in hardware.196 */197 if (!hisi_validate_event_group(event))198 return -EINVAL;199 200 hisi_pmu = to_hisi_pmu(event->pmu);201 if (event->attr.config > hisi_pmu->check_event)202 return -EINVAL;203 204 if (hisi_pmu->on_cpu == -1)205 return -EINVAL;206 /*207 * We don't assign an index until we actually place the event onto208 * hardware. Use -1 to signify that we haven't decided where to put it209 * yet.210 */211 hwc->idx = -1;212 hwc->config_base = event->attr.config;213 214 if (hisi_pmu->ops->check_filter && hisi_pmu->ops->check_filter(event))215 return -EINVAL;216 217 /* Enforce to use the same CPU for all events in this PMU */218 event->cpu = hisi_pmu->on_cpu;219 220 return 0;221}222EXPORT_SYMBOL_GPL(hisi_uncore_pmu_event_init);223 224/*225 * Set the counter to count the event that we're interested in,226 * and enable interrupt and counter.227 */228static void hisi_uncore_pmu_enable_event(struct perf_event *event)229{230 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);231 struct hw_perf_event *hwc = &event->hw;232 233 hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,234 HISI_GET_EVENTID(event));235 236 if (hisi_pmu->ops->enable_filter)237 hisi_pmu->ops->enable_filter(event);238 239 hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);240 hisi_pmu->ops->enable_counter(hisi_pmu, hwc);241}242 243/*244 * Disable counter and interrupt.245 */246static void hisi_uncore_pmu_disable_event(struct perf_event *event)247{248 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);249 struct hw_perf_event *hwc = &event->hw;250 251 hisi_pmu->ops->disable_counter(hisi_pmu, hwc);252 hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);253 254 if (hisi_pmu->ops->disable_filter)255 hisi_pmu->ops->disable_filter(event);256}257 258void hisi_uncore_pmu_set_event_period(struct perf_event *event)259{260 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);261 struct hw_perf_event *hwc = &event->hw;262 263 /*264 * The HiSilicon PMU counters support 32 bits or 48 bits, depending on265 * the PMU. We reduce it to 2^(counter_bits - 1) to account for the266 * extreme interrupt latency. So we could hopefully handle the overflow267 * interrupt before another 2^(counter_bits - 1) events occur and the268 * counter overtakes its previous value.269 */270 u64 val = BIT_ULL(hisi_pmu->counter_bits - 1);271 272 local64_set(&hwc->prev_count, val);273 /* Write start value to the hardware event counter */274 hisi_pmu->ops->write_counter(hisi_pmu, hwc, val);275}276EXPORT_SYMBOL_GPL(hisi_uncore_pmu_set_event_period);277 278void hisi_uncore_pmu_event_update(struct perf_event *event)279{280 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);281 struct hw_perf_event *hwc = &event->hw;282 u64 delta, prev_raw_count, new_raw_count;283 284 do {285 /* Read the count from the counter register */286 new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc);287 prev_raw_count = local64_read(&hwc->prev_count);288 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,289 new_raw_count) != prev_raw_count);290 /*291 * compute the delta292 */293 delta = (new_raw_count - prev_raw_count) &294 HISI_MAX_PERIOD(hisi_pmu->counter_bits);295 local64_add(delta, &event->count);296}297EXPORT_SYMBOL_GPL(hisi_uncore_pmu_event_update);298 299void hisi_uncore_pmu_start(struct perf_event *event, int flags)300{301 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);302 struct hw_perf_event *hwc = &event->hw;303 304 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))305 return;306 307 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));308 hwc->state = 0;309 hisi_uncore_pmu_set_event_period(event);310 311 if (flags & PERF_EF_RELOAD) {312 u64 prev_raw_count = local64_read(&hwc->prev_count);313 314 hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count);315 }316 317 hisi_uncore_pmu_enable_event(event);318 perf_event_update_userpage(event);319}320EXPORT_SYMBOL_GPL(hisi_uncore_pmu_start);321 322void hisi_uncore_pmu_stop(struct perf_event *event, int flags)323{324 struct hw_perf_event *hwc = &event->hw;325 326 hisi_uncore_pmu_disable_event(event);327 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);328 hwc->state |= PERF_HES_STOPPED;329 330 if (hwc->state & PERF_HES_UPTODATE)331 return;332 333 /* Read hardware counter and update the perf counter statistics */334 hisi_uncore_pmu_event_update(event);335 hwc->state |= PERF_HES_UPTODATE;336}337EXPORT_SYMBOL_GPL(hisi_uncore_pmu_stop);338 339int hisi_uncore_pmu_add(struct perf_event *event, int flags)340{341 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);342 struct hw_perf_event *hwc = &event->hw;343 int idx;344 345 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;346 347 /* Get an available counter index for counting */348 idx = hisi_pmu->ops->get_event_idx(event);349 if (idx < 0)350 return idx;351 352 event->hw.idx = idx;353 hisi_pmu->pmu_events.hw_events[idx] = event;354 355 if (flags & PERF_EF_START)356 hisi_uncore_pmu_start(event, PERF_EF_RELOAD);357 358 return 0;359}360EXPORT_SYMBOL_GPL(hisi_uncore_pmu_add);361 362void hisi_uncore_pmu_del(struct perf_event *event, int flags)363{364 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);365 struct hw_perf_event *hwc = &event->hw;366 367 hisi_uncore_pmu_stop(event, PERF_EF_UPDATE);368 hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx);369 perf_event_update_userpage(event);370 hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL;371}372EXPORT_SYMBOL_GPL(hisi_uncore_pmu_del);373 374void hisi_uncore_pmu_read(struct perf_event *event)375{376 /* Read hardware counter and update the perf counter statistics */377 hisi_uncore_pmu_event_update(event);378}379EXPORT_SYMBOL_GPL(hisi_uncore_pmu_read);380 381void hisi_uncore_pmu_enable(struct pmu *pmu)382{383 struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);384 bool enabled = !bitmap_empty(hisi_pmu->pmu_events.used_mask,385 hisi_pmu->num_counters);386 387 if (!enabled)388 return;389 390 hisi_pmu->ops->start_counters(hisi_pmu);391}392EXPORT_SYMBOL_GPL(hisi_uncore_pmu_enable);393 394void hisi_uncore_pmu_disable(struct pmu *pmu)395{396 struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);397 398 hisi_pmu->ops->stop_counters(hisi_pmu);399}400EXPORT_SYMBOL_GPL(hisi_uncore_pmu_disable);401 402 403/*404 * The Super CPU Cluster (SCCL) and CPU Cluster (CCL) IDs can be405 * determined from the MPIDR_EL1, but the encoding varies by CPU:406 *407 * - For MT variants of TSV110:408 * SCCL is Aff2[7:3], CCL is Aff2[2:0]409 *410 * - For other MT parts:411 * SCCL is Aff3[7:0], CCL is Aff2[7:0]412 *413 * - For non-MT parts:414 * SCCL is Aff2[7:0], CCL is Aff1[7:0]415 */416static void hisi_read_sccl_and_ccl_id(int *scclp, int *cclp)417{418 u64 mpidr = read_cpuid_mpidr();419 int aff3 = MPIDR_AFFINITY_LEVEL(mpidr, 3);420 int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2);421 int aff1 = MPIDR_AFFINITY_LEVEL(mpidr, 1);422 bool mt = mpidr & MPIDR_MT_BITMASK;423 int sccl, ccl;424 425 if (mt && read_cpuid_part_number() == HISI_CPU_PART_TSV110) {426 sccl = aff2 >> 3;427 ccl = aff2 & 0x7;428 } else if (mt) {429 sccl = aff3;430 ccl = aff2;431 } else {432 sccl = aff2;433 ccl = aff1;434 }435 436 if (scclp)437 *scclp = sccl;438 if (cclp)439 *cclp = ccl;440}441 442/*443 * Check whether the CPU is associated with this uncore PMU444 */445static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu)446{447 int sccl_id, ccl_id;448 449 /* If SCCL_ID is -1, the PMU is in a SICL and has no CPU affinity */450 if (hisi_pmu->sccl_id == -1)451 return true;452 453 if (hisi_pmu->ccl_id == -1) {454 /* If CCL_ID is -1, the PMU only shares the same SCCL */455 hisi_read_sccl_and_ccl_id(&sccl_id, NULL);456 457 return sccl_id == hisi_pmu->sccl_id;458 }459 460 hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id);461 462 return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id;463}464 465int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)466{467 struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,468 node);469 470 if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu))471 return 0;472 473 cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus);474 475 /* If another CPU is already managing this PMU, simply return. */476 if (hisi_pmu->on_cpu != -1)477 return 0;478 479 /* Use this CPU in cpumask for event counting */480 hisi_pmu->on_cpu = cpu;481 482 /* Overflow interrupt also should use the same CPU */483 WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));484 485 return 0;486}487EXPORT_SYMBOL_GPL(hisi_uncore_pmu_online_cpu);488 489int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)490{491 struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,492 node);493 unsigned int target;494 495 if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus))496 return 0;497 498 /* Nothing to do if this CPU doesn't own the PMU */499 if (hisi_pmu->on_cpu != cpu)500 return 0;501 502 /* Give up ownership of the PMU */503 hisi_pmu->on_cpu = -1;504 505 /* Choose a new CPU to migrate ownership of the PMU to */506 target = cpumask_any_and_but(&hisi_pmu->associated_cpus,507 cpu_online_mask, cpu);508 if (target >= nr_cpu_ids)509 return 0;510 511 perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);512 /* Use this CPU for event counting */513 hisi_pmu->on_cpu = target;514 WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));515 516 return 0;517}518EXPORT_SYMBOL_GPL(hisi_uncore_pmu_offline_cpu);519 520void hisi_pmu_init(struct hisi_pmu *hisi_pmu, struct module *module)521{522 struct pmu *pmu = &hisi_pmu->pmu;523 524 pmu->module = module;525 pmu->parent = hisi_pmu->dev;526 pmu->task_ctx_nr = perf_invalid_context;527 pmu->event_init = hisi_uncore_pmu_event_init;528 pmu->pmu_enable = hisi_uncore_pmu_enable;529 pmu->pmu_disable = hisi_uncore_pmu_disable;530 pmu->add = hisi_uncore_pmu_add;531 pmu->del = hisi_uncore_pmu_del;532 pmu->start = hisi_uncore_pmu_start;533 pmu->stop = hisi_uncore_pmu_stop;534 pmu->read = hisi_uncore_pmu_read;535 pmu->attr_groups = hisi_pmu->pmu_events.attr_groups;536 pmu->capabilities = PERF_PMU_CAP_NO_EXCLUDE;537}538EXPORT_SYMBOL_GPL(hisi_pmu_init);539 540MODULE_DESCRIPTION("HiSilicon SoC uncore Performance Monitor driver framework");541MODULE_LICENSE("GPL v2");542