824 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Alibaba DDR Sub-System Driveway PMU driver4 *5 * Copyright (C) 2022 Alibaba Inc6 */7 8#define ALI_DRW_PMUNAME "ali_drw"9#define ALI_DRW_DRVNAME ALI_DRW_PMUNAME "_pmu"10#define pr_fmt(fmt) ALI_DRW_DRVNAME ": " fmt11 12#include <linux/acpi.h>13#include <linux/bitfield.h>14#include <linux/bitmap.h>15#include <linux/bitops.h>16#include <linux/cpuhotplug.h>17#include <linux/cpumask.h>18#include <linux/device.h>19#include <linux/errno.h>20#include <linux/interrupt.h>21#include <linux/irq.h>22#include <linux/kernel.h>23#include <linux/list.h>24#include <linux/module.h>25#include <linux/mutex.h>26#include <linux/perf_event.h>27#include <linux/platform_device.h>28#include <linux/printk.h>29#include <linux/rculist.h>30#include <linux/refcount.h>31 32 33#define ALI_DRW_PMU_COMMON_MAX_COUNTERS 1634#define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE 1935 36#define ALI_DRW_PMU_PA_SHIFT 1237#define ALI_DRW_PMU_CNT_INIT 0x0000000038#define ALI_DRW_CNT_MAX_PERIOD 0xffffffff39#define ALI_DRW_PMU_CYCLE_EVT_ID 0x8040 41#define ALI_DRW_PMU_CNT_CTRL 0xC0042#define ALI_DRW_PMU_CNT_RST BIT(2)43#define ALI_DRW_PMU_CNT_STOP BIT(1)44#define ALI_DRW_PMU_CNT_START BIT(0)45 46#define ALI_DRW_PMU_CNT_STATE 0xC0447#define ALI_DRW_PMU_TEST_CTRL 0xC0848#define ALI_DRW_PMU_CNT_PRELOAD 0xC0C49 50#define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK GENMASK(23, 0)51#define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK GENMASK(31, 0)52#define ALI_DRW_PMU_CYCLE_CNT_HIGH 0xC1053#define ALI_DRW_PMU_CYCLE_CNT_LOW 0xC1454 55/* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */56#define ALI_DRW_PMU_EVENT_SEL0 0xC6857/* counter 0-3 use sel0, counter 4-7 use sel1...*/58#define ALI_DRW_PMU_EVENT_SELn(n) \59 (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)60#define ALI_DRW_PMCOM_CNT_EN BIT(7)61#define ALI_DRW_PMCOM_CNT_EVENT_MASK GENMASK(5, 0)62#define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \63 (8 * (n % 4))64 65/* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */66#define ALI_DRW_PMU_COMMON_COUNTER0 0xC7867#define ALI_DRW_PMU_COMMON_COUNTERn(n) \68 (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))69 70#define ALI_DRW_PMU_OV_INTR_ENABLE_CTL 0xCB871#define ALI_DRW_PMU_OV_INTR_DISABLE_CTL 0xCBC72#define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS 0xCC073#define ALI_DRW_PMU_OV_INTR_CLR 0xCC474#define ALI_DRW_PMU_OV_INTR_STATUS 0xCC875#define ALI_DRW_PMCOM_CNT_OV_INTR_MASK GENMASK(23, 8)76#define ALI_DRW_PMBW_CNT_OV_INTR_MASK GENMASK(7, 0)77#define ALI_DRW_PMU_OV_INTR_MASK GENMASK_ULL(63, 0)78 79static int ali_drw_cpuhp_state_num;80 81static LIST_HEAD(ali_drw_pmu_irqs);82static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);83 84struct ali_drw_pmu_irq {85 struct hlist_node node;86 struct list_head irqs_node;87 struct list_head pmus_node;88 int irq_num;89 int cpu;90 refcount_t refcount;91};92 93struct ali_drw_pmu {94 void __iomem *cfg_base;95 struct device *dev;96 97 struct list_head pmus_node;98 struct ali_drw_pmu_irq *irq;99 int irq_num;100 int cpu;101 DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);102 struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];103 int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];104 105 struct pmu pmu;106};107 108#define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))109 110#define DRW_CONFIG_EVENTID GENMASK(7, 0)111#define GET_DRW_EVENTID(event) FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)112 113static ssize_t ali_drw_pmu_format_show(struct device *dev,114 struct device_attribute *attr, char *buf)115{116 struct dev_ext_attribute *eattr;117 118 eattr = container_of(attr, struct dev_ext_attribute, attr);119 120 return sprintf(buf, "%s\n", (char *)eattr->var);121}122 123/*124 * PMU event attributes125 */126static ssize_t ali_drw_pmu_event_show(struct device *dev,127 struct device_attribute *attr, char *page)128{129 struct dev_ext_attribute *eattr;130 131 eattr = container_of(attr, struct dev_ext_attribute, attr);132 133 return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);134}135 136#define ALI_DRW_PMU_ATTR(_name, _func, _config) \137 (&((struct dev_ext_attribute[]) { \138 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \139 })[0].attr.attr)140 141#define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \142 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)143#define ALI_DRW_PMU_EVENT_ATTR(_name, _config) \144 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)145 146static struct attribute *ali_drw_pmu_events_attrs[] = {147 ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr, 0x0),148 ALI_DRW_PMU_EVENT_ATTR(hif_wr, 0x1),149 ALI_DRW_PMU_EVENT_ATTR(hif_rd, 0x2),150 ALI_DRW_PMU_EVENT_ATTR(hif_rmw, 0x3),151 ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd, 0x4),152 ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles, 0x7),153 ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles, 0x8),154 ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical, 0x9),155 ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical, 0xA),156 ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical, 0xB),157 ALI_DRW_PMU_EVENT_ATTR(op_is_activate, 0xC),158 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr, 0xD),159 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate, 0xE),160 ALI_DRW_PMU_EVENT_ATTR(op_is_rd, 0xF),161 ALI_DRW_PMU_EVENT_ATTR(op_is_wr, 0x10),162 ALI_DRW_PMU_EVENT_ATTR(op_is_mwr, 0x11),163 ALI_DRW_PMU_EVENT_ATTR(op_is_precharge, 0x12),164 ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr, 0x13),165 ALI_DRW_PMU_EVENT_ATTR(precharge_for_other, 0x14),166 ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions, 0x15),167 ALI_DRW_PMU_EVENT_ATTR(write_combine, 0x16),168 ALI_DRW_PMU_EVENT_ATTR(war_hazard, 0x17),169 ALI_DRW_PMU_EVENT_ATTR(raw_hazard, 0x18),170 ALI_DRW_PMU_EVENT_ATTR(waw_hazard, 0x19),171 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0, 0x1A),172 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1, 0x1B),173 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2, 0x1C),174 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3, 0x1D),175 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0, 0x1E),176 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1, 0x1F),177 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2, 0x20),178 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3, 0x21),179 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0, 0x26),180 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1, 0x27),181 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2, 0x28),182 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3, 0x29),183 ALI_DRW_PMU_EVENT_ATTR(op_is_refresh, 0x2A),184 ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref, 0x2B),185 ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode, 0x2D),186 ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl, 0x2E),187 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),188 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),189 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc, 0x34),190 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr, 0x35),191 ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr, 0x36),192 ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart, 0x37),193 ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch, 0x38),194 ALI_DRW_PMU_EVENT_ATTR(chi_txreq, 0x39),195 ALI_DRW_PMU_EVENT_ATTR(chi_txdat, 0x3A),196 ALI_DRW_PMU_EVENT_ATTR(chi_rxdat, 0x3B),197 ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp, 0x3C),198 ALI_DRW_PMU_EVENT_ATTR(tsz_vio, 0x3D),199 ALI_DRW_PMU_EVENT_ATTR(cycle, 0x80),200 NULL,201};202 203static struct attribute_group ali_drw_pmu_events_attr_group = {204 .name = "events",205 .attrs = ali_drw_pmu_events_attrs,206};207 208static struct attribute *ali_drw_pmu_format_attr[] = {209 ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),210 NULL,211};212 213static const struct attribute_group ali_drw_pmu_format_group = {214 .name = "format",215 .attrs = ali_drw_pmu_format_attr,216};217 218static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,219 struct device_attribute *attr,220 char *buf)221{222 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));223 224 return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));225}226 227static struct device_attribute ali_drw_pmu_cpumask_attr =228 __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);229 230static struct attribute *ali_drw_pmu_cpumask_attrs[] = {231 &ali_drw_pmu_cpumask_attr.attr,232 NULL,233};234 235static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {236 .attrs = ali_drw_pmu_cpumask_attrs,237};238 239static umode_t ali_drw_pmu_identifier_attr_visible(struct kobject *kobj,240 struct attribute *attr, int n)241{242 return attr->mode;243}244 245static DEVICE_STRING_ATTR_RO(ali_drw_pmu_identifier, 0444, "ali_drw_pmu");246 247static struct attribute *ali_drw_pmu_identifier_attrs[] = {248 &dev_attr_ali_drw_pmu_identifier.attr.attr,249 NULL250};251 252static const struct attribute_group ali_drw_pmu_identifier_attr_group = {253 .attrs = ali_drw_pmu_identifier_attrs,254 .is_visible = ali_drw_pmu_identifier_attr_visible255};256 257static const struct attribute_group *ali_drw_pmu_attr_groups[] = {258 &ali_drw_pmu_events_attr_group,259 &ali_drw_pmu_cpumask_attr_group,260 &ali_drw_pmu_format_group,261 &ali_drw_pmu_identifier_attr_group,262 NULL,263};264 265/* find a counter for event, then in add func, hw.idx will equal to counter */266static int ali_drw_get_counter_idx(struct perf_event *event)267{268 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);269 int idx;270 271 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {272 if (!test_and_set_bit(idx, drw_pmu->used_mask))273 return idx;274 }275 276 /* The counters are all in use. */277 return -EBUSY;278}279 280static u64 ali_drw_pmu_read_counter(struct perf_event *event)281{282 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);283 u64 cycle_high, cycle_low;284 285 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {286 cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);287 cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;288 cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);289 cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;290 return (cycle_high << 32 | cycle_low);291 }292 293 return readl(drw_pmu->cfg_base +294 ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));295}296 297static void ali_drw_pmu_event_update(struct perf_event *event)298{299 struct hw_perf_event *hwc = &event->hw;300 u64 delta, prev, now;301 302 do {303 prev = local64_read(&hwc->prev_count);304 now = ali_drw_pmu_read_counter(event);305 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);306 307 /* handle overflow. */308 delta = now - prev;309 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)310 delta &= ALI_DRW_PMU_OV_INTR_MASK;311 else312 delta &= ALI_DRW_CNT_MAX_PERIOD;313 local64_add(delta, &event->count);314}315 316static void ali_drw_pmu_event_set_period(struct perf_event *event)317{318 u64 pre_val;319 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);320 321 /* set a preload counter for test purpose */322 writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,323 drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);324 325 /* set conunter initial value */326 pre_val = ALI_DRW_PMU_CNT_INIT;327 writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);328 local64_set(&event->hw.prev_count, pre_val);329 330 /* set sel mode to zero to start test */331 writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);332}333 334static void ali_drw_pmu_enable_counter(struct perf_event *event)335{336 u32 val, subval, reg, shift;337 int counter = event->hw.idx;338 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);339 340 reg = ALI_DRW_PMU_EVENT_SELn(counter);341 val = readl(drw_pmu->cfg_base + reg);342 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |343 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);344 345 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);346 val &= ~(GENMASK(7, 0) << shift);347 val |= subval << shift;348 349 writel(val, drw_pmu->cfg_base + reg);350}351 352static void ali_drw_pmu_disable_counter(struct perf_event *event)353{354 u32 val, reg, subval, shift;355 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);356 int counter = event->hw.idx;357 358 reg = ALI_DRW_PMU_EVENT_SELn(counter);359 val = readl(drw_pmu->cfg_base + reg);360 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |361 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);362 363 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);364 val &= ~(GENMASK(7, 0) << shift);365 val |= subval << shift;366 367 writel(val, drw_pmu->cfg_base + reg);368}369 370static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)371{372 struct ali_drw_pmu_irq *irq = data;373 struct ali_drw_pmu *drw_pmu;374 irqreturn_t ret = IRQ_NONE;375 376 rcu_read_lock();377 list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {378 unsigned long status, clr_status;379 struct perf_event *event;380 unsigned int idx;381 382 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {383 event = drw_pmu->events[idx];384 if (!event)385 continue;386 ali_drw_pmu_disable_counter(event);387 }388 389 /* common counter intr status */390 status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);391 status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);392 if (status) {393 for_each_set_bit(idx, &status,394 ALI_DRW_PMU_COMMON_MAX_COUNTERS) {395 event = drw_pmu->events[idx];396 if (WARN_ON_ONCE(!event))397 continue;398 ali_drw_pmu_event_update(event);399 ali_drw_pmu_event_set_period(event);400 }401 402 /* clear common counter intr status */403 clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);404 writel(clr_status,405 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);406 }407 408 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {409 event = drw_pmu->events[idx];410 if (!event)411 continue;412 if (!(event->hw.state & PERF_HES_STOPPED))413 ali_drw_pmu_enable_counter(event);414 }415 if (status)416 ret = IRQ_HANDLED;417 }418 rcu_read_unlock();419 return ret;420}421 422static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device423 *pdev, int irq_num)424{425 int ret;426 struct ali_drw_pmu_irq *irq;427 428 list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {429 if (irq->irq_num == irq_num430 && refcount_inc_not_zero(&irq->refcount))431 return irq;432 }433 434 irq = kzalloc(sizeof(*irq), GFP_KERNEL);435 if (!irq)436 return ERR_PTR(-ENOMEM);437 438 INIT_LIST_HEAD(&irq->pmus_node);439 440 /* Pick one CPU to be the preferred one to use */441 irq->cpu = smp_processor_id();442 refcount_set(&irq->refcount, 1);443 444 /*445 * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same446 * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers447 * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not448 * share with other component.449 */450 ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,451 IRQF_SHARED, dev_name(&pdev->dev), irq);452 if (ret < 0) {453 dev_err(&pdev->dev,454 "Fail to request IRQ:%d ret:%d\n", irq_num, ret);455 goto out_free;456 }457 458 ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));459 if (ret)460 goto out_free;461 462 ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,463 &irq->node);464 if (ret)465 goto out_free;466 467 irq->irq_num = irq_num;468 list_add(&irq->irqs_node, &ali_drw_pmu_irqs);469 470 return irq;471 472out_free:473 kfree(irq);474 return ERR_PTR(ret);475}476 477static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,478 struct platform_device *pdev)479{480 int irq_num;481 struct ali_drw_pmu_irq *irq;482 483 /* Read and init IRQ */484 irq_num = platform_get_irq(pdev, 0);485 if (irq_num < 0)486 return irq_num;487 488 mutex_lock(&ali_drw_pmu_irqs_lock);489 irq = __ali_drw_pmu_init_irq(pdev, irq_num);490 mutex_unlock(&ali_drw_pmu_irqs_lock);491 492 if (IS_ERR(irq))493 return PTR_ERR(irq);494 495 drw_pmu->irq = irq;496 497 mutex_lock(&ali_drw_pmu_irqs_lock);498 list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);499 mutex_unlock(&ali_drw_pmu_irqs_lock);500 501 return 0;502}503 504static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)505{506 struct ali_drw_pmu_irq *irq = drw_pmu->irq;507 508 mutex_lock(&ali_drw_pmu_irqs_lock);509 list_del_rcu(&drw_pmu->pmus_node);510 511 if (!refcount_dec_and_test(&irq->refcount)) {512 mutex_unlock(&ali_drw_pmu_irqs_lock);513 return;514 }515 516 list_del(&irq->irqs_node);517 mutex_unlock(&ali_drw_pmu_irqs_lock);518 519 WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));520 cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,521 &irq->node);522 kfree(irq);523}524 525static int ali_drw_pmu_event_init(struct perf_event *event)526{527 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);528 struct hw_perf_event *hwc = &event->hw;529 struct perf_event *sibling;530 struct device *dev = drw_pmu->pmu.dev;531 532 if (event->attr.type != event->pmu->type)533 return -ENOENT;534 535 if (is_sampling_event(event)) {536 dev_err(dev, "Sampling not supported!\n");537 return -EOPNOTSUPP;538 }539 540 if (event->attach_state & PERF_ATTACH_TASK) {541 dev_err(dev, "Per-task counter cannot allocate!\n");542 return -EOPNOTSUPP;543 }544 545 event->cpu = drw_pmu->cpu;546 if (event->cpu < 0) {547 dev_err(dev, "Per-task mode not supported!\n");548 return -EOPNOTSUPP;549 }550 551 if (event->group_leader != event &&552 !is_software_event(event->group_leader)) {553 dev_err(dev, "driveway only allow one event!\n");554 return -EINVAL;555 }556 557 for_each_sibling_event(sibling, event->group_leader) {558 if (sibling != event && !is_software_event(sibling)) {559 dev_err(dev, "driveway event not allowed!\n");560 return -EINVAL;561 }562 }563 564 /* reset all the pmu counters */565 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);566 567 hwc->idx = -1;568 569 return 0;570}571 572static void ali_drw_pmu_start(struct perf_event *event, int flags)573{574 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);575 576 event->hw.state = 0;577 578 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {579 writel(ALI_DRW_PMU_CNT_START,580 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);581 return;582 }583 584 ali_drw_pmu_event_set_period(event);585 if (flags & PERF_EF_RELOAD) {586 unsigned long prev_raw_count =587 local64_read(&event->hw.prev_count);588 writel(prev_raw_count,589 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);590 }591 592 ali_drw_pmu_enable_counter(event);593 594 writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);595}596 597static void ali_drw_pmu_stop(struct perf_event *event, int flags)598{599 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);600 601 if (event->hw.state & PERF_HES_STOPPED)602 return;603 604 if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)605 ali_drw_pmu_disable_counter(event);606 607 writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);608 609 ali_drw_pmu_event_update(event);610 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;611}612 613static int ali_drw_pmu_add(struct perf_event *event, int flags)614{615 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);616 struct hw_perf_event *hwc = &event->hw;617 int idx = -1;618 int evtid;619 620 evtid = GET_DRW_EVENTID(event);621 622 if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {623 idx = ali_drw_get_counter_idx(event);624 if (idx < 0)625 return idx;626 drw_pmu->events[idx] = event;627 drw_pmu->evtids[idx] = evtid;628 }629 hwc->idx = idx;630 631 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;632 633 if (flags & PERF_EF_START)634 ali_drw_pmu_start(event, PERF_EF_RELOAD);635 636 /* Propagate our changes to the userspace mapping. */637 perf_event_update_userpage(event);638 639 return 0;640}641 642static void ali_drw_pmu_del(struct perf_event *event, int flags)643{644 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);645 struct hw_perf_event *hwc = &event->hw;646 int idx = hwc->idx;647 648 ali_drw_pmu_stop(event, PERF_EF_UPDATE);649 650 if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {651 drw_pmu->events[idx] = NULL;652 drw_pmu->evtids[idx] = 0;653 clear_bit(idx, drw_pmu->used_mask);654 }655 656 perf_event_update_userpage(event);657}658 659static void ali_drw_pmu_read(struct perf_event *event)660{661 ali_drw_pmu_event_update(event);662}663 664static int ali_drw_pmu_probe(struct platform_device *pdev)665{666 struct ali_drw_pmu *drw_pmu;667 struct resource *res;668 char *name;669 int ret;670 671 drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);672 if (!drw_pmu)673 return -ENOMEM;674 675 drw_pmu->dev = &pdev->dev;676 platform_set_drvdata(pdev, drw_pmu);677 678 drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);679 if (IS_ERR(drw_pmu->cfg_base))680 return PTR_ERR(drw_pmu->cfg_base);681 682 name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",683 (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));684 if (!name)685 return -ENOMEM;686 687 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);688 689 /* enable the generation of interrupt by all common counters */690 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,691 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);692 693 /* clearing interrupt status */694 writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);695 696 drw_pmu->cpu = smp_processor_id();697 698 ret = ali_drw_pmu_init_irq(drw_pmu, pdev);699 if (ret)700 return ret;701 702 drw_pmu->pmu = (struct pmu) {703 .module = THIS_MODULE,704 .parent = &pdev->dev,705 .task_ctx_nr = perf_invalid_context,706 .event_init = ali_drw_pmu_event_init,707 .add = ali_drw_pmu_add,708 .del = ali_drw_pmu_del,709 .start = ali_drw_pmu_start,710 .stop = ali_drw_pmu_stop,711 .read = ali_drw_pmu_read,712 .attr_groups = ali_drw_pmu_attr_groups,713 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,714 };715 716 ret = perf_pmu_register(&drw_pmu->pmu, name, -1);717 if (ret) {718 dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");719 ali_drw_pmu_uninit_irq(drw_pmu);720 }721 722 return ret;723}724 725static void ali_drw_pmu_remove(struct platform_device *pdev)726{727 struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);728 729 /* disable the generation of interrupt by all common counters */730 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,731 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);732 733 ali_drw_pmu_uninit_irq(drw_pmu);734 perf_pmu_unregister(&drw_pmu->pmu);735}736 737static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)738{739 struct ali_drw_pmu_irq *irq;740 struct ali_drw_pmu *drw_pmu;741 unsigned int target;742 743 irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);744 if (cpu != irq->cpu)745 return 0;746 747 target = cpumask_any_and_but(cpumask_of_node(cpu_to_node(cpu)),748 cpu_online_mask, cpu);749 if (target >= nr_cpu_ids)750 target = cpumask_any_but(cpu_online_mask, cpu);751 752 if (target >= nr_cpu_ids)753 return 0;754 755 /* We're only reading, but this isn't the place to be involving RCU */756 mutex_lock(&ali_drw_pmu_irqs_lock);757 list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)758 perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);759 mutex_unlock(&ali_drw_pmu_irqs_lock);760 761 WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));762 irq->cpu = target;763 764 return 0;765}766 767/*768 * Due to historical reasons, the HID used in the production environment is769 * ARMHD700, so we leave ARMHD700 as Compatible ID.770 */771static const struct acpi_device_id ali_drw_acpi_match[] = {772 {"BABA5000", 0},773 {"ARMHD700", 0},774 {}775};776 777MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);778 779static struct platform_driver ali_drw_pmu_driver = {780 .driver = {781 .name = "ali_drw_pmu",782 .acpi_match_table = ali_drw_acpi_match,783 },784 .probe = ali_drw_pmu_probe,785 .remove_new = ali_drw_pmu_remove,786};787 788static int __init ali_drw_pmu_init(void)789{790 int ret;791 792 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,793 "ali_drw_pmu:online",794 NULL, ali_drw_pmu_offline_cpu);795 796 if (ret < 0) {797 pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",798 ret);799 return ret;800 }801 ali_drw_cpuhp_state_num = ret;802 803 ret = platform_driver_register(&ali_drw_pmu_driver);804 if (ret)805 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);806 807 return ret;808}809 810static void __exit ali_drw_pmu_exit(void)811{812 platform_driver_unregister(&ali_drw_pmu_driver);813 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);814}815 816module_init(ali_drw_pmu_init);817module_exit(ali_drw_pmu_exit);818 819MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>");820MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>");821MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");822MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");823MODULE_LICENSE("GPL v2");824