1008 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * This driver adds support for PCIe PMU RCiEP device. Related4 * perf events are bandwidth, latency etc.5 *6 * Copyright (C) 2021 HiSilicon Limited7 * Author: Qi Liu <liuqi115@huawei.com>8 */9#include <linux/bitfield.h>10#include <linux/bitmap.h>11#include <linux/bug.h>12#include <linux/device.h>13#include <linux/err.h>14#include <linux/interrupt.h>15#include <linux/irq.h>16#include <linux/kernel.h>17#include <linux/list.h>18#include <linux/module.h>19#include <linux/pci.h>20#include <linux/perf_event.h>21 22#define DRV_NAME "hisi_pcie_pmu"23/* Define registers */24#define HISI_PCIE_GLOBAL_CTRL 0x0025#define HISI_PCIE_EVENT_CTRL 0x01026#define HISI_PCIE_CNT 0x09027#define HISI_PCIE_EXT_CNT 0x11028#define HISI_PCIE_INT_STAT 0x15029#define HISI_PCIE_INT_MASK 0x15430#define HISI_PCIE_REG_BDF 0xfe031#define HISI_PCIE_REG_VERSION 0xfe432#define HISI_PCIE_REG_INFO 0xfe833 34/* Define command in HISI_PCIE_GLOBAL_CTRL */35#define HISI_PCIE_GLOBAL_EN 0x0136#define HISI_PCIE_GLOBAL_NONE 037 38/* Define command in HISI_PCIE_EVENT_CTRL */39#define HISI_PCIE_EVENT_EN BIT_ULL(20)40#define HISI_PCIE_RESET_CNT BIT_ULL(22)41#define HISI_PCIE_INIT_SET BIT_ULL(34)42#define HISI_PCIE_THR_EN BIT_ULL(26)43#define HISI_PCIE_TARGET_EN BIT_ULL(32)44#define HISI_PCIE_TRIG_EN BIT_ULL(52)45 46/* Define offsets in HISI_PCIE_EVENT_CTRL */47#define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0)48#define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27)49#define HISI_PCIE_THR_M GENMASK_ULL(31, 28)50#define HISI_PCIE_LEN_M GENMASK_ULL(35, 34)51#define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36)52#define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53)53#define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56)54 55/* Default config of TLP length mode, will count both TLP headers and payloads */56#define HISI_PCIE_LEN_M_DEFAULT 3ULL57 58#define HISI_PCIE_MAX_COUNTERS 859#define HISI_PCIE_REG_STEP 860#define HISI_PCIE_THR_MAX_VAL 1061#define HISI_PCIE_TRIG_MAX_VAL 1062#define HISI_PCIE_MAX_PERIOD (GENMASK_ULL(63, 0))63#define HISI_PCIE_INIT_VAL BIT_ULL(63)64 65struct hisi_pcie_pmu {66 struct perf_event *hw_events[HISI_PCIE_MAX_COUNTERS];67 struct hlist_node node;68 struct pci_dev *pdev;69 struct pmu pmu;70 void __iomem *base;71 int irq;72 u32 identifier;73 /* Minimum and maximum BDF of root ports monitored by PMU */74 u16 bdf_min;75 u16 bdf_max;76 int on_cpu;77};78 79struct hisi_pcie_reg_pair {80 u16 lo;81 u16 hi;82};83 84#define to_pcie_pmu(p) (container_of((p), struct hisi_pcie_pmu, pmu))85#define GET_PCI_DEVFN(bdf) ((bdf) & 0xff)86 87#define HISI_PCIE_PMU_FILTER_ATTR(_name, _config, _hi, _lo) \88 static u64 hisi_pcie_get_##_name(struct perf_event *event) \89 { \90 return FIELD_GET(GENMASK(_hi, _lo), event->attr._config); \91 } \92 93HISI_PCIE_PMU_FILTER_ATTR(event, config, 16, 0);94HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0);95HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4);96HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5);97HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9);98HISI_PCIE_PMU_FILTER_ATTR(len_mode, config1, 11, 10);99HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0);100HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16);101 102static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attribute *attr,103 char *buf)104{105 struct perf_pmu_events_attr *pmu_attr =106 container_of(attr, struct perf_pmu_events_attr, attr);107 108 return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id);109}110 111#define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format) \112 (&((struct dev_ext_attribute[]){ \113 { .attr = __ATTR(_name, 0444, device_show_string, NULL), \114 .var = (void *)_format } \115 })[0].attr.attr)116 117#define HISI_PCIE_PMU_EVENT_ATTR(_name, _id) \118 PMU_EVENT_ATTR_ID(_name, hisi_pcie_event_sysfs_show, _id)119 120static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf)121{122 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));123 124 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));125}126static DEVICE_ATTR_RO(cpumask);127 128static ssize_t identifier_show(struct device *dev, struct device_attribute *attr, char *buf)129{130 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));131 132 return sysfs_emit(buf, "%#x\n", pcie_pmu->identifier);133}134static DEVICE_ATTR_RO(identifier);135 136static ssize_t bus_show(struct device *dev, struct device_attribute *attr, char *buf)137{138 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));139 140 return sysfs_emit(buf, "%#04x\n", PCI_BUS_NUM(pcie_pmu->bdf_min));141}142static DEVICE_ATTR_RO(bus);143 144static ssize_t bdf_min_show(struct device *dev, struct device_attribute *attr, char *buf)145{146 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));147 148 return sysfs_emit(buf, "%#04x\n", pcie_pmu->bdf_min);149}150static DEVICE_ATTR_RO(bdf_min);151 152static ssize_t bdf_max_show(struct device *dev, struct device_attribute *attr, char *buf)153{154 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));155 156 return sysfs_emit(buf, "%#04x\n", pcie_pmu->bdf_max);157}158static DEVICE_ATTR_RO(bdf_max);159 160static struct hisi_pcie_reg_pair161hisi_pcie_parse_reg_value(struct hisi_pcie_pmu *pcie_pmu, u32 reg_off)162{163 u32 val = readl_relaxed(pcie_pmu->base + reg_off);164 struct hisi_pcie_reg_pair regs = {165 .lo = val,166 .hi = val >> 16,167 };168 169 return regs;170}171 172/*173 * Hardware counter and ext_counter work together for bandwidth, latency, bus174 * utilization and buffer occupancy events. For example, RX memory write latency175 * events(index = 0x0010), counter counts total delay cycles and ext_counter176 * counts RX memory write PCIe packets number.177 *178 * As we don't want PMU driver to process these two data, "delay cycles" can179 * be treated as an independent event(index = 0x0010), "RX memory write packets180 * number" as another(index = 0x10010). BIT 16 is used to distinguish and 0-15181 * bits are "real" event index, which can be used to set HISI_PCIE_EVENT_CTRL.182 */183#define EXT_COUNTER_IS_USED(idx) ((idx) & BIT(16))184 185static u32 hisi_pcie_get_real_event(struct perf_event *event)186{187 return hisi_pcie_get_event(event) & GENMASK(15, 0);188}189 190static u32 hisi_pcie_pmu_get_offset(u32 offset, u32 idx)191{192 return offset + HISI_PCIE_REG_STEP * idx;193}194 195static u32 hisi_pcie_pmu_readl(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset,196 u32 idx)197{198 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);199 200 return readl_relaxed(pcie_pmu->base + offset);201}202 203static void hisi_pcie_pmu_writel(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u32 val)204{205 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);206 207 writel_relaxed(val, pcie_pmu->base + offset);208}209 210static u64 hisi_pcie_pmu_readq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx)211{212 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);213 214 return readq_relaxed(pcie_pmu->base + offset);215}216 217static void hisi_pcie_pmu_writeq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u64 val)218{219 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);220 221 writeq_relaxed(val, pcie_pmu->base + offset);222}223 224static u64 hisi_pcie_pmu_get_event_ctrl_val(struct perf_event *event)225{226 u64 port, trig_len, thr_len, len_mode;227 u64 reg = 0;228 229 /* Config HISI_PCIE_EVENT_CTRL according to event. */230 reg |= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event));231 232 /* Config HISI_PCIE_EVENT_CTRL according to root port or EP device. */233 port = hisi_pcie_get_port(event);234 if (port)235 reg |= FIELD_PREP(HISI_PCIE_TARGET_M, port);236 else237 reg |= HISI_PCIE_TARGET_EN |238 FIELD_PREP(HISI_PCIE_TARGET_M, hisi_pcie_get_bdf(event));239 240 /* Config HISI_PCIE_EVENT_CTRL according to trigger condition. */241 trig_len = hisi_pcie_get_trig_len(event);242 if (trig_len) {243 reg |= FIELD_PREP(HISI_PCIE_TRIG_M, trig_len);244 reg |= FIELD_PREP(HISI_PCIE_TRIG_MODE_M, hisi_pcie_get_trig_mode(event));245 reg |= HISI_PCIE_TRIG_EN;246 }247 248 /* Config HISI_PCIE_EVENT_CTRL according to threshold condition. */249 thr_len = hisi_pcie_get_thr_len(event);250 if (thr_len) {251 reg |= FIELD_PREP(HISI_PCIE_THR_M, thr_len);252 reg |= FIELD_PREP(HISI_PCIE_THR_MODE_M, hisi_pcie_get_thr_mode(event));253 reg |= HISI_PCIE_THR_EN;254 }255 256 len_mode = hisi_pcie_get_len_mode(event);257 if (len_mode)258 reg |= FIELD_PREP(HISI_PCIE_LEN_M, len_mode);259 else260 reg |= FIELD_PREP(HISI_PCIE_LEN_M, HISI_PCIE_LEN_M_DEFAULT);261 262 return reg;263}264 265static void hisi_pcie_pmu_config_event_ctrl(struct perf_event *event)266{267 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);268 struct hw_perf_event *hwc = &event->hw;269 u64 reg = hisi_pcie_pmu_get_event_ctrl_val(event);270 271 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg);272}273 274static void hisi_pcie_pmu_clear_event_ctrl(struct perf_event *event)275{276 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);277 struct hw_perf_event *hwc = &event->hw;278 279 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, HISI_PCIE_INIT_SET);280}281 282static bool hisi_pcie_pmu_valid_requester_id(struct hisi_pcie_pmu *pcie_pmu, u32 bdf)283{284 struct pci_dev *root_port, *pdev;285 u16 rp_bdf;286 287 pdev = pci_get_domain_bus_and_slot(pci_domain_nr(pcie_pmu->pdev->bus), PCI_BUS_NUM(bdf),288 GET_PCI_DEVFN(bdf));289 if (!pdev)290 return false;291 292 root_port = pcie_find_root_port(pdev);293 if (!root_port) {294 pci_dev_put(pdev);295 return false;296 }297 298 pci_dev_put(pdev);299 rp_bdf = pci_dev_id(root_port);300 return rp_bdf >= pcie_pmu->bdf_min && rp_bdf <= pcie_pmu->bdf_max;301}302 303static bool hisi_pcie_pmu_valid_filter(struct perf_event *event,304 struct hisi_pcie_pmu *pcie_pmu)305{306 u32 requester_id = hisi_pcie_get_bdf(event);307 308 if (hisi_pcie_get_thr_len(event) > HISI_PCIE_THR_MAX_VAL)309 return false;310 311 if (hisi_pcie_get_trig_len(event) > HISI_PCIE_TRIG_MAX_VAL)312 return false;313 314 /* Need to explicitly set filter of "port" or "bdf" */315 if (!hisi_pcie_get_port(event) &&316 !hisi_pcie_pmu_valid_requester_id(pcie_pmu, requester_id))317 return false;318 319 return true;320}321 322/*323 * Check Whether two events share the same config. The same config means not324 * only the event code, but also the filter settings of the two events are325 * the same.326 */327static bool hisi_pcie_pmu_cmp_event(struct perf_event *target,328 struct perf_event *event)329{330 return hisi_pcie_pmu_get_event_ctrl_val(target) ==331 hisi_pcie_pmu_get_event_ctrl_val(event);332}333 334static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event)335{336 struct perf_event *sibling, *leader = event->group_leader;337 struct perf_event *event_group[HISI_PCIE_MAX_COUNTERS];338 int counters = 1;339 int num;340 341 event_group[0] = leader;342 if (!is_software_event(leader)) {343 if (leader->pmu != event->pmu)344 return false;345 346 if (leader != event && !hisi_pcie_pmu_cmp_event(leader, event))347 event_group[counters++] = event;348 }349 350 for_each_sibling_event(sibling, event->group_leader) {351 if (is_software_event(sibling))352 continue;353 354 if (sibling->pmu != event->pmu)355 return false;356 357 for (num = 0; num < counters; num++) {358 /*359 * If we find a related event, then it's a valid group360 * since we don't need to allocate a new counter for it.361 */362 if (hisi_pcie_pmu_cmp_event(event_group[num], sibling))363 break;364 }365 366 /*367 * Otherwise it's a new event but if there's no available counter,368 * fail the check since we cannot schedule all the events in369 * the group simultaneously.370 */371 if (num == HISI_PCIE_MAX_COUNTERS)372 return false;373 374 if (num == counters)375 event_group[counters++] = sibling;376 }377 378 return true;379}380 381static int hisi_pcie_pmu_event_init(struct perf_event *event)382{383 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);384 struct hw_perf_event *hwc = &event->hw;385 386 /* Check the type first before going on, otherwise it's not our event */387 if (event->attr.type != event->pmu->type)388 return -ENOENT;389 390 if (EXT_COUNTER_IS_USED(hisi_pcie_get_event(event)))391 hwc->event_base = HISI_PCIE_EXT_CNT;392 else393 hwc->event_base = HISI_PCIE_CNT;394 395 /* Sampling is not supported. */396 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)397 return -EOPNOTSUPP;398 399 if (!hisi_pcie_pmu_valid_filter(event, pcie_pmu))400 return -EINVAL;401 402 if (!hisi_pcie_pmu_validate_event_group(event))403 return -EINVAL;404 405 event->cpu = pcie_pmu->on_cpu;406 407 return 0;408}409 410static u64 hisi_pcie_pmu_read_counter(struct perf_event *event)411{412 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);413 u32 idx = event->hw.idx;414 415 return hisi_pcie_pmu_readq(pcie_pmu, event->hw.event_base, idx);416}417 418/*419 * Check all work events, if a relevant event is found then we return it420 * first, otherwise return the first idle counter (need to reset).421 */422static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu,423 struct perf_event *event)424{425 int first_idle = -EAGAIN;426 struct perf_event *sibling;427 int idx;428 429 for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) {430 sibling = pcie_pmu->hw_events[idx];431 if (!sibling) {432 if (first_idle == -EAGAIN)433 first_idle = idx;434 continue;435 }436 437 /* Related events must be used in group */438 if (hisi_pcie_pmu_cmp_event(sibling, event) &&439 sibling->group_leader == event->group_leader)440 return idx;441 }442 443 return first_idle;444}445 446static void hisi_pcie_pmu_event_update(struct perf_event *event)447{448 struct hw_perf_event *hwc = &event->hw;449 u64 new_cnt, prev_cnt, delta;450 451 do {452 prev_cnt = local64_read(&hwc->prev_count);453 new_cnt = hisi_pcie_pmu_read_counter(event);454 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt,455 new_cnt) != prev_cnt);456 457 delta = (new_cnt - prev_cnt) & HISI_PCIE_MAX_PERIOD;458 local64_add(delta, &event->count);459}460 461static void hisi_pcie_pmu_read(struct perf_event *event)462{463 hisi_pcie_pmu_event_update(event);464}465 466static void hisi_pcie_pmu_set_period(struct perf_event *event)467{468 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);469 struct hw_perf_event *hwc = &event->hw;470 int idx = hwc->idx;471 u64 orig_cnt, cnt;472 473 orig_cnt = hisi_pcie_pmu_read_counter(event);474 475 local64_set(&hwc->prev_count, HISI_PCIE_INIT_VAL);476 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_CNT, idx, HISI_PCIE_INIT_VAL);477 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EXT_CNT, idx, HISI_PCIE_INIT_VAL);478 479 /*480 * The counter maybe unwritable if the target event is unsupported.481 * Check this by comparing the counts after setting the period. If482 * the counts stay unchanged after setting the period then update483 * the hwc->prev_count correctly. Otherwise the final counts user484 * get maybe totally wrong.485 */486 cnt = hisi_pcie_pmu_read_counter(event);487 if (orig_cnt == cnt)488 local64_set(&hwc->prev_count, cnt);489}490 491static void hisi_pcie_pmu_enable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)492{493 u32 idx = hwc->idx;494 u64 val;495 496 val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx);497 val |= HISI_PCIE_EVENT_EN;498 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val);499}500 501static void hisi_pcie_pmu_disable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)502{503 u32 idx = hwc->idx;504 u64 val;505 506 val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx);507 val &= ~HISI_PCIE_EVENT_EN;508 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val);509}510 511static void hisi_pcie_pmu_enable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)512{513 u32 idx = hwc->idx;514 515 hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 0);516}517 518static void hisi_pcie_pmu_disable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)519{520 u32 idx = hwc->idx;521 522 hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 1);523}524 525static void hisi_pcie_pmu_reset_counter(struct hisi_pcie_pmu *pcie_pmu, int idx)526{527 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_RESET_CNT);528 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_INIT_SET);529}530 531static void hisi_pcie_pmu_start(struct perf_event *event, int flags)532{533 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);534 struct hw_perf_event *hwc = &event->hw;535 int idx = hwc->idx;536 u64 prev_cnt;537 538 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))539 return;540 541 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));542 hwc->state = 0;543 544 hisi_pcie_pmu_config_event_ctrl(event);545 hisi_pcie_pmu_enable_counter(pcie_pmu, hwc);546 hisi_pcie_pmu_enable_int(pcie_pmu, hwc);547 hisi_pcie_pmu_set_period(event);548 549 if (flags & PERF_EF_RELOAD) {550 prev_cnt = local64_read(&hwc->prev_count);551 hisi_pcie_pmu_writeq(pcie_pmu, hwc->event_base, idx, prev_cnt);552 }553 554 perf_event_update_userpage(event);555}556 557static void hisi_pcie_pmu_stop(struct perf_event *event, int flags)558{559 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);560 struct hw_perf_event *hwc = &event->hw;561 562 hisi_pcie_pmu_event_update(event);563 hisi_pcie_pmu_disable_int(pcie_pmu, hwc);564 hisi_pcie_pmu_disable_counter(pcie_pmu, hwc);565 hisi_pcie_pmu_clear_event_ctrl(event);566 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);567 hwc->state |= PERF_HES_STOPPED;568 569 if (hwc->state & PERF_HES_UPTODATE)570 return;571 572 hwc->state |= PERF_HES_UPTODATE;573}574 575static int hisi_pcie_pmu_add(struct perf_event *event, int flags)576{577 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);578 struct hw_perf_event *hwc = &event->hw;579 int idx;580 581 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;582 583 idx = hisi_pcie_pmu_get_event_idx(pcie_pmu, event);584 if (idx < 0)585 return idx;586 587 hwc->idx = idx;588 589 /* No enabled counter found with related event, reset it */590 if (!pcie_pmu->hw_events[idx]) {591 hisi_pcie_pmu_reset_counter(pcie_pmu, idx);592 pcie_pmu->hw_events[idx] = event;593 }594 595 if (flags & PERF_EF_START)596 hisi_pcie_pmu_start(event, PERF_EF_RELOAD);597 598 return 0;599}600 601static void hisi_pcie_pmu_del(struct perf_event *event, int flags)602{603 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);604 struct hw_perf_event *hwc = &event->hw;605 606 hisi_pcie_pmu_stop(event, PERF_EF_UPDATE);607 pcie_pmu->hw_events[hwc->idx] = NULL;608 perf_event_update_userpage(event);609}610 611static void hisi_pcie_pmu_enable(struct pmu *pmu)612{613 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu);614 int num;615 616 for (num = 0; num < HISI_PCIE_MAX_COUNTERS; num++) {617 if (pcie_pmu->hw_events[num])618 break;619 }620 621 if (num == HISI_PCIE_MAX_COUNTERS)622 return;623 624 writel(HISI_PCIE_GLOBAL_EN, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL);625}626 627static void hisi_pcie_pmu_disable(struct pmu *pmu)628{629 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu);630 631 writel(HISI_PCIE_GLOBAL_NONE, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL);632}633 634static irqreturn_t hisi_pcie_pmu_irq(int irq, void *data)635{636 struct hisi_pcie_pmu *pcie_pmu = data;637 irqreturn_t ret = IRQ_NONE;638 struct perf_event *event;639 u32 overflown;640 int idx;641 642 for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) {643 overflown = hisi_pcie_pmu_readl(pcie_pmu, HISI_PCIE_INT_STAT, idx);644 if (!overflown)645 continue;646 647 /* Clear status of interrupt. */648 hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_STAT, idx, 1);649 event = pcie_pmu->hw_events[idx];650 if (!event)651 continue;652 653 hisi_pcie_pmu_event_update(event);654 hisi_pcie_pmu_set_period(event);655 ret = IRQ_HANDLED;656 }657 658 return ret;659}660 661static int hisi_pcie_pmu_irq_register(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)662{663 int irq, ret;664 665 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);666 if (ret < 0) {667 pci_err(pdev, "Failed to enable MSI vectors: %d\n", ret);668 return ret;669 }670 671 irq = pci_irq_vector(pdev, 0);672 ret = request_irq(irq, hisi_pcie_pmu_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME,673 pcie_pmu);674 if (ret) {675 pci_err(pdev, "Failed to register IRQ: %d\n", ret);676 pci_free_irq_vectors(pdev);677 return ret;678 }679 680 pcie_pmu->irq = irq;681 682 return 0;683}684 685static void hisi_pcie_pmu_irq_unregister(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)686{687 free_irq(pcie_pmu->irq, pcie_pmu);688 pci_free_irq_vectors(pdev);689}690 691static int hisi_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)692{693 struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node);694 695 if (pcie_pmu->on_cpu == -1) {696 pcie_pmu->on_cpu = cpumask_local_spread(0, dev_to_node(&pcie_pmu->pdev->dev));697 WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(pcie_pmu->on_cpu)));698 }699 700 return 0;701}702 703static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)704{705 struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node);706 unsigned int target;707 int numa_node;708 709 /* Nothing to do if this CPU doesn't own the PMU */710 if (pcie_pmu->on_cpu != cpu)711 return 0;712 713 pcie_pmu->on_cpu = -1;714 715 /* Choose a local CPU from all online cpus. */716 numa_node = dev_to_node(&pcie_pmu->pdev->dev);717 718 target = cpumask_any_and_but(cpumask_of_node(numa_node),719 cpu_online_mask, cpu);720 if (target >= nr_cpu_ids)721 target = cpumask_any_but(cpu_online_mask, cpu);722 723 if (target >= nr_cpu_ids) {724 pci_err(pcie_pmu->pdev, "There is no CPU to set\n");725 return 0;726 }727 728 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);729 /* Use this CPU for event counting */730 pcie_pmu->on_cpu = target;731 WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(target)));732 733 return 0;734}735 736static struct attribute *hisi_pcie_pmu_events_attr[] = {737 HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_latency, 0x0010),738 HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_cnt, 0x10010),739 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_latency, 0x0210),740 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210),741 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011),742 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011),743 HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_flux, 0x0104),744 HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_time, 0x10104),745 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x0804),746 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x10804),747 HISI_PCIE_PMU_EVENT_ATTR(rx_cpl_flux, 0x2004),748 HISI_PCIE_PMU_EVENT_ATTR(rx_cpl_time, 0x12004),749 HISI_PCIE_PMU_EVENT_ATTR(tx_mwr_flux, 0x0105),750 HISI_PCIE_PMU_EVENT_ATTR(tx_mwr_time, 0x10105),751 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x0405),752 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x10405),753 HISI_PCIE_PMU_EVENT_ATTR(tx_cpl_flux, 0x1005),754 HISI_PCIE_PMU_EVENT_ATTR(tx_cpl_time, 0x11005),755 NULL756};757 758static struct attribute_group hisi_pcie_pmu_events_group = {759 .name = "events",760 .attrs = hisi_pcie_pmu_events_attr,761};762 763static struct attribute *hisi_pcie_pmu_format_attr[] = {764 HISI_PCIE_PMU_FORMAT_ATTR(event, "config:0-16"),765 HISI_PCIE_PMU_FORMAT_ATTR(thr_len, "config1:0-3"),766 HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4"),767 HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8"),768 HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9"),769 HISI_PCIE_PMU_FORMAT_ATTR(len_mode, "config1:10-11"),770 HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15"),771 HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31"),772 NULL773};774 775static const struct attribute_group hisi_pcie_pmu_format_group = {776 .name = "format",777 .attrs = hisi_pcie_pmu_format_attr,778};779 780static struct attribute *hisi_pcie_pmu_bus_attrs[] = {781 &dev_attr_bus.attr,782 &dev_attr_bdf_max.attr,783 &dev_attr_bdf_min.attr,784 NULL785};786 787static const struct attribute_group hisi_pcie_pmu_bus_attr_group = {788 .attrs = hisi_pcie_pmu_bus_attrs,789};790 791static struct attribute *hisi_pcie_pmu_cpumask_attrs[] = {792 &dev_attr_cpumask.attr,793 NULL794};795 796static const struct attribute_group hisi_pcie_pmu_cpumask_attr_group = {797 .attrs = hisi_pcie_pmu_cpumask_attrs,798};799 800static struct attribute *hisi_pcie_pmu_identifier_attrs[] = {801 &dev_attr_identifier.attr,802 NULL803};804 805static const struct attribute_group hisi_pcie_pmu_identifier_attr_group = {806 .attrs = hisi_pcie_pmu_identifier_attrs,807};808 809static const struct attribute_group *hisi_pcie_pmu_attr_groups[] = {810 &hisi_pcie_pmu_events_group,811 &hisi_pcie_pmu_format_group,812 &hisi_pcie_pmu_bus_attr_group,813 &hisi_pcie_pmu_cpumask_attr_group,814 &hisi_pcie_pmu_identifier_attr_group,815 NULL816};817 818static int hisi_pcie_alloc_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)819{820 struct hisi_pcie_reg_pair regs;821 u16 sicl_id, core_id;822 char *name;823 824 regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_BDF);825 pcie_pmu->bdf_min = regs.lo;826 pcie_pmu->bdf_max = regs.hi;827 828 regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_INFO);829 sicl_id = regs.hi;830 core_id = regs.lo;831 832 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_pcie%u_core%u", sicl_id, core_id);833 if (!name)834 return -ENOMEM;835 836 pcie_pmu->pdev = pdev;837 pcie_pmu->on_cpu = -1;838 pcie_pmu->identifier = readl(pcie_pmu->base + HISI_PCIE_REG_VERSION);839 pcie_pmu->pmu = (struct pmu) {840 .name = name,841 .module = THIS_MODULE,842 .parent = &pdev->dev,843 .event_init = hisi_pcie_pmu_event_init,844 .pmu_enable = hisi_pcie_pmu_enable,845 .pmu_disable = hisi_pcie_pmu_disable,846 .add = hisi_pcie_pmu_add,847 .del = hisi_pcie_pmu_del,848 .start = hisi_pcie_pmu_start,849 .stop = hisi_pcie_pmu_stop,850 .read = hisi_pcie_pmu_read,851 .task_ctx_nr = perf_invalid_context,852 .attr_groups = hisi_pcie_pmu_attr_groups,853 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,854 };855 856 return 0;857}858 859static int hisi_pcie_init_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)860{861 int ret;862 863 pcie_pmu->base = pci_ioremap_bar(pdev, 2);864 if (!pcie_pmu->base) {865 pci_err(pdev, "Ioremap failed for pcie_pmu resource\n");866 return -ENOMEM;867 }868 869 ret = hisi_pcie_alloc_pmu(pdev, pcie_pmu);870 if (ret)871 goto err_iounmap;872 873 ret = hisi_pcie_pmu_irq_register(pdev, pcie_pmu);874 if (ret)875 goto err_iounmap;876 877 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node);878 if (ret) {879 pci_err(pdev, "Failed to register hotplug: %d\n", ret);880 goto err_irq_unregister;881 }882 883 ret = perf_pmu_register(&pcie_pmu->pmu, pcie_pmu->pmu.name, -1);884 if (ret) {885 pci_err(pdev, "Failed to register PCIe PMU: %d\n", ret);886 goto err_hotplug_unregister;887 }888 889 return ret;890 891err_hotplug_unregister:892 cpuhp_state_remove_instance_nocalls(893 CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node);894 895err_irq_unregister:896 hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu);897 898err_iounmap:899 iounmap(pcie_pmu->base);900 901 return ret;902}903 904static void hisi_pcie_uninit_pmu(struct pci_dev *pdev)905{906 struct hisi_pcie_pmu *pcie_pmu = pci_get_drvdata(pdev);907 908 perf_pmu_unregister(&pcie_pmu->pmu);909 cpuhp_state_remove_instance_nocalls(910 CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node);911 hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu);912 iounmap(pcie_pmu->base);913}914 915static int hisi_pcie_init_dev(struct pci_dev *pdev)916{917 int ret;918 919 ret = pcim_enable_device(pdev);920 if (ret) {921 pci_err(pdev, "Failed to enable PCI device: %d\n", ret);922 return ret;923 }924 925 ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME);926 if (ret < 0) {927 pci_err(pdev, "Failed to request PCI mem regions: %d\n", ret);928 return ret;929 }930 931 pci_set_master(pdev);932 933 return 0;934}935 936static int hisi_pcie_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id)937{938 struct hisi_pcie_pmu *pcie_pmu;939 int ret;940 941 pcie_pmu = devm_kzalloc(&pdev->dev, sizeof(*pcie_pmu), GFP_KERNEL);942 if (!pcie_pmu)943 return -ENOMEM;944 945 ret = hisi_pcie_init_dev(pdev);946 if (ret)947 return ret;948 949 ret = hisi_pcie_init_pmu(pdev, pcie_pmu);950 if (ret)951 return ret;952 953 pci_set_drvdata(pdev, pcie_pmu);954 955 return ret;956}957 958static void hisi_pcie_pmu_remove(struct pci_dev *pdev)959{960 hisi_pcie_uninit_pmu(pdev);961 pci_set_drvdata(pdev, NULL);962}963 964static const struct pci_device_id hisi_pcie_pmu_ids[] = {965 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12d) },966 { 0, }967};968MODULE_DEVICE_TABLE(pci, hisi_pcie_pmu_ids);969 970static struct pci_driver hisi_pcie_pmu_driver = {971 .name = DRV_NAME,972 .id_table = hisi_pcie_pmu_ids,973 .probe = hisi_pcie_pmu_probe,974 .remove = hisi_pcie_pmu_remove,975};976 977static int __init hisi_pcie_module_init(void)978{979 int ret;980 981 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE,982 "AP_PERF_ARM_HISI_PCIE_PMU_ONLINE",983 hisi_pcie_pmu_online_cpu,984 hisi_pcie_pmu_offline_cpu);985 if (ret) {986 pr_err("Failed to setup PCIe PMU hotplug: %d\n", ret);987 return ret;988 }989 990 ret = pci_register_driver(&hisi_pcie_pmu_driver);991 if (ret)992 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE);993 994 return ret;995}996module_init(hisi_pcie_module_init);997 998static void __exit hisi_pcie_module_exit(void)999{1000 pci_unregister_driver(&hisi_pcie_pmu_driver);1001 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE);1002}1003module_exit(hisi_pcie_module_exit);1004 1005MODULE_DESCRIPTION("HiSilicon PCIe PMU driver");1006MODULE_LICENSE("GPL v2");1007MODULE_AUTHOR("Qi Liu <liuqi115@huawei.com>");1008