1675 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * This driver adds support for HNS3 PMU iEP device. Related perf events are4 * bandwidth, latency, packet rate, interrupt rate etc.5 *6 * Copyright (C) 2022 HiSilicon Limited7 */8#include <linux/bitfield.h>9#include <linux/bitmap.h>10#include <linux/bug.h>11#include <linux/cpuhotplug.h>12#include <linux/cpumask.h>13#include <linux/delay.h>14#include <linux/device.h>15#include <linux/err.h>16#include <linux/interrupt.h>17#include <linux/iopoll.h>18#include <linux/io-64-nonatomic-hi-lo.h>19#include <linux/irq.h>20#include <linux/kernel.h>21#include <linux/list.h>22#include <linux/module.h>23#include <linux/pci.h>24#include <linux/pci-epf.h>25#include <linux/perf_event.h>26#include <linux/smp.h>27 28/* registers offset address */29#define HNS3_PMU_REG_GLOBAL_CTRL 0x000030#define HNS3_PMU_REG_CLOCK_FREQ 0x002031#define HNS3_PMU_REG_BDF 0x0fe032#define HNS3_PMU_REG_VERSION 0x0fe433#define HNS3_PMU_REG_DEVICE_ID 0x0fe834 35#define HNS3_PMU_REG_EVENT_OFFSET 0x100036#define HNS3_PMU_REG_EVENT_SIZE 0x100037#define HNS3_PMU_REG_EVENT_CTRL_LOW 0x0038#define HNS3_PMU_REG_EVENT_CTRL_HIGH 0x0439#define HNS3_PMU_REG_EVENT_INTR_STATUS 0x0840#define HNS3_PMU_REG_EVENT_INTR_MASK 0x0c41#define HNS3_PMU_REG_EVENT_COUNTER 0x1042#define HNS3_PMU_REG_EVENT_EXT_COUNTER 0x1843#define HNS3_PMU_REG_EVENT_QID_CTRL 0x2844#define HNS3_PMU_REG_EVENT_QID_PARA 0x2c45 46#define HNS3_PMU_FILTER_SUPPORT_GLOBAL BIT(0)47#define HNS3_PMU_FILTER_SUPPORT_PORT BIT(1)48#define HNS3_PMU_FILTER_SUPPORT_PORT_TC BIT(2)49#define HNS3_PMU_FILTER_SUPPORT_FUNC BIT(3)50#define HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE BIT(4)51#define HNS3_PMU_FILTER_SUPPORT_FUNC_INTR BIT(5)52 53#define HNS3_PMU_FILTER_ALL_TC 0xf54#define HNS3_PMU_FILTER_ALL_QUEUE 0xffff55 56#define HNS3_PMU_CTRL_SUBEVENT_S 457#define HNS3_PMU_CTRL_FILTER_MODE_S 2458 59#define HNS3_PMU_GLOBAL_START BIT(0)60 61#define HNS3_PMU_EVENT_STATUS_RESET BIT(11)62#define HNS3_PMU_EVENT_EN BIT(12)63#define HNS3_PMU_EVENT_OVERFLOW_RESTART BIT(15)64 65#define HNS3_PMU_QID_PARA_FUNC_S 066#define HNS3_PMU_QID_PARA_QUEUE_S 1667 68#define HNS3_PMU_QID_CTRL_REQ_ENABLE BIT(0)69#define HNS3_PMU_QID_CTRL_DONE BIT(1)70#define HNS3_PMU_QID_CTRL_MISS BIT(2)71 72#define HNS3_PMU_INTR_MASK_OVERFLOW BIT(1)73 74#define HNS3_PMU_MAX_HW_EVENTS 875 76/*77 * Each hardware event contains two registers (counter and ext_counter) for78 * bandwidth, packet rate, latency and interrupt rate. These two registers will79 * be triggered to run at the same when a hardware event is enabled. The meaning80 * of counter and ext_counter of different event type are different, their81 * meaning show as follow:82 *83 * +----------------+------------------+---------------+84 * | event type | counter | ext_counter |85 * +----------------+------------------+---------------+86 * | bandwidth | byte number | cycle number |87 * +----------------+------------------+---------------+88 * | packet rate | packet number | cycle number |89 * +----------------+------------------+---------------+90 * | latency | cycle number | packet number |91 * +----------------+------------------+---------------+92 * | interrupt rate | interrupt number | cycle number |93 * +----------------+------------------+---------------+94 *95 * The cycle number indicates increment of counter of hardware timer, the96 * frequency of hardware timer can be read from hw_clk_freq file.97 *98 * Performance of each hardware event is calculated by: counter / ext_counter.99 *100 * Since processing of data is preferred to be done in userspace, we expose101 * ext_counter as a separate event for userspace and use bit 16 to indicate it.102 * For example, event 0x00001 and 0x10001 are actually one event for hardware103 * because bit 0-15 are same. If the bit 16 of one event is 0 means to read104 * counter register, otherwise means to read ext_counter register.105 */106/* bandwidth events */107#define HNS3_PMU_EVT_BW_SSU_EGU_BYTE_NUM 0x00001108#define HNS3_PMU_EVT_BW_SSU_EGU_TIME 0x10001109#define HNS3_PMU_EVT_BW_SSU_RPU_BYTE_NUM 0x00002110#define HNS3_PMU_EVT_BW_SSU_RPU_TIME 0x10002111#define HNS3_PMU_EVT_BW_SSU_ROCE_BYTE_NUM 0x00003112#define HNS3_PMU_EVT_BW_SSU_ROCE_TIME 0x10003113#define HNS3_PMU_EVT_BW_ROCE_SSU_BYTE_NUM 0x00004114#define HNS3_PMU_EVT_BW_ROCE_SSU_TIME 0x10004115#define HNS3_PMU_EVT_BW_TPU_SSU_BYTE_NUM 0x00005116#define HNS3_PMU_EVT_BW_TPU_SSU_TIME 0x10005117#define HNS3_PMU_EVT_BW_RPU_RCBRX_BYTE_NUM 0x00006118#define HNS3_PMU_EVT_BW_RPU_RCBRX_TIME 0x10006119#define HNS3_PMU_EVT_BW_RCBTX_TXSCH_BYTE_NUM 0x00008120#define HNS3_PMU_EVT_BW_RCBTX_TXSCH_TIME 0x10008121#define HNS3_PMU_EVT_BW_WR_FBD_BYTE_NUM 0x00009122#define HNS3_PMU_EVT_BW_WR_FBD_TIME 0x10009123#define HNS3_PMU_EVT_BW_WR_EBD_BYTE_NUM 0x0000a124#define HNS3_PMU_EVT_BW_WR_EBD_TIME 0x1000a125#define HNS3_PMU_EVT_BW_RD_FBD_BYTE_NUM 0x0000b126#define HNS3_PMU_EVT_BW_RD_FBD_TIME 0x1000b127#define HNS3_PMU_EVT_BW_RD_EBD_BYTE_NUM 0x0000c128#define HNS3_PMU_EVT_BW_RD_EBD_TIME 0x1000c129#define HNS3_PMU_EVT_BW_RD_PAY_M0_BYTE_NUM 0x0000d130#define HNS3_PMU_EVT_BW_RD_PAY_M0_TIME 0x1000d131#define HNS3_PMU_EVT_BW_RD_PAY_M1_BYTE_NUM 0x0000e132#define HNS3_PMU_EVT_BW_RD_PAY_M1_TIME 0x1000e133#define HNS3_PMU_EVT_BW_WR_PAY_M0_BYTE_NUM 0x0000f134#define HNS3_PMU_EVT_BW_WR_PAY_M0_TIME 0x1000f135#define HNS3_PMU_EVT_BW_WR_PAY_M1_BYTE_NUM 0x00010136#define HNS3_PMU_EVT_BW_WR_PAY_M1_TIME 0x10010137 138/* packet rate events */139#define HNS3_PMU_EVT_PPS_IGU_SSU_PACKET_NUM 0x00100140#define HNS3_PMU_EVT_PPS_IGU_SSU_TIME 0x10100141#define HNS3_PMU_EVT_PPS_SSU_EGU_PACKET_NUM 0x00101142#define HNS3_PMU_EVT_PPS_SSU_EGU_TIME 0x10101143#define HNS3_PMU_EVT_PPS_SSU_RPU_PACKET_NUM 0x00102144#define HNS3_PMU_EVT_PPS_SSU_RPU_TIME 0x10102145#define HNS3_PMU_EVT_PPS_SSU_ROCE_PACKET_NUM 0x00103146#define HNS3_PMU_EVT_PPS_SSU_ROCE_TIME 0x10103147#define HNS3_PMU_EVT_PPS_ROCE_SSU_PACKET_NUM 0x00104148#define HNS3_PMU_EVT_PPS_ROCE_SSU_TIME 0x10104149#define HNS3_PMU_EVT_PPS_TPU_SSU_PACKET_NUM 0x00105150#define HNS3_PMU_EVT_PPS_TPU_SSU_TIME 0x10105151#define HNS3_PMU_EVT_PPS_RPU_RCBRX_PACKET_NUM 0x00106152#define HNS3_PMU_EVT_PPS_RPU_RCBRX_TIME 0x10106153#define HNS3_PMU_EVT_PPS_RCBTX_TPU_PACKET_NUM 0x00107154#define HNS3_PMU_EVT_PPS_RCBTX_TPU_TIME 0x10107155#define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_PACKET_NUM 0x00108156#define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_TIME 0x10108157#define HNS3_PMU_EVT_PPS_WR_FBD_PACKET_NUM 0x00109158#define HNS3_PMU_EVT_PPS_WR_FBD_TIME 0x10109159#define HNS3_PMU_EVT_PPS_WR_EBD_PACKET_NUM 0x0010a160#define HNS3_PMU_EVT_PPS_WR_EBD_TIME 0x1010a161#define HNS3_PMU_EVT_PPS_RD_FBD_PACKET_NUM 0x0010b162#define HNS3_PMU_EVT_PPS_RD_FBD_TIME 0x1010b163#define HNS3_PMU_EVT_PPS_RD_EBD_PACKET_NUM 0x0010c164#define HNS3_PMU_EVT_PPS_RD_EBD_TIME 0x1010c165#define HNS3_PMU_EVT_PPS_RD_PAY_M0_PACKET_NUM 0x0010d166#define HNS3_PMU_EVT_PPS_RD_PAY_M0_TIME 0x1010d167#define HNS3_PMU_EVT_PPS_RD_PAY_M1_PACKET_NUM 0x0010e168#define HNS3_PMU_EVT_PPS_RD_PAY_M1_TIME 0x1010e169#define HNS3_PMU_EVT_PPS_WR_PAY_M0_PACKET_NUM 0x0010f170#define HNS3_PMU_EVT_PPS_WR_PAY_M0_TIME 0x1010f171#define HNS3_PMU_EVT_PPS_WR_PAY_M1_PACKET_NUM 0x00110172#define HNS3_PMU_EVT_PPS_WR_PAY_M1_TIME 0x10110173#define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_PACKET_NUM 0x00111174#define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_TIME 0x10111175#define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_PACKET_NUM 0x00112176#define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_TIME 0x10112177 178/* latency events */179#define HNS3_PMU_EVT_DLY_TX_PUSH_TIME 0x00202180#define HNS3_PMU_EVT_DLY_TX_PUSH_PACKET_NUM 0x10202181#define HNS3_PMU_EVT_DLY_TX_TIME 0x00204182#define HNS3_PMU_EVT_DLY_TX_PACKET_NUM 0x10204183#define HNS3_PMU_EVT_DLY_SSU_TX_NIC_TIME 0x00206184#define HNS3_PMU_EVT_DLY_SSU_TX_NIC_PACKET_NUM 0x10206185#define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_TIME 0x00207186#define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_PACKET_NUM 0x10207187#define HNS3_PMU_EVT_DLY_SSU_RX_NIC_TIME 0x00208188#define HNS3_PMU_EVT_DLY_SSU_RX_NIC_PACKET_NUM 0x10208189#define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_TIME 0x00209190#define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_PACKET_NUM 0x10209191#define HNS3_PMU_EVT_DLY_RPU_TIME 0x0020e192#define HNS3_PMU_EVT_DLY_RPU_PACKET_NUM 0x1020e193#define HNS3_PMU_EVT_DLY_TPU_TIME 0x0020f194#define HNS3_PMU_EVT_DLY_TPU_PACKET_NUM 0x1020f195#define HNS3_PMU_EVT_DLY_RPE_TIME 0x00210196#define HNS3_PMU_EVT_DLY_RPE_PACKET_NUM 0x10210197#define HNS3_PMU_EVT_DLY_TPE_TIME 0x00211198#define HNS3_PMU_EVT_DLY_TPE_PACKET_NUM 0x10211199#define HNS3_PMU_EVT_DLY_TPE_PUSH_TIME 0x00212200#define HNS3_PMU_EVT_DLY_TPE_PUSH_PACKET_NUM 0x10212201#define HNS3_PMU_EVT_DLY_WR_FBD_TIME 0x00213202#define HNS3_PMU_EVT_DLY_WR_FBD_PACKET_NUM 0x10213203#define HNS3_PMU_EVT_DLY_WR_EBD_TIME 0x00214204#define HNS3_PMU_EVT_DLY_WR_EBD_PACKET_NUM 0x10214205#define HNS3_PMU_EVT_DLY_RD_FBD_TIME 0x00215206#define HNS3_PMU_EVT_DLY_RD_FBD_PACKET_NUM 0x10215207#define HNS3_PMU_EVT_DLY_RD_EBD_TIME 0x00216208#define HNS3_PMU_EVT_DLY_RD_EBD_PACKET_NUM 0x10216209#define HNS3_PMU_EVT_DLY_RD_PAY_M0_TIME 0x00217210#define HNS3_PMU_EVT_DLY_RD_PAY_M0_PACKET_NUM 0x10217211#define HNS3_PMU_EVT_DLY_RD_PAY_M1_TIME 0x00218212#define HNS3_PMU_EVT_DLY_RD_PAY_M1_PACKET_NUM 0x10218213#define HNS3_PMU_EVT_DLY_WR_PAY_M0_TIME 0x00219214#define HNS3_PMU_EVT_DLY_WR_PAY_M0_PACKET_NUM 0x10219215#define HNS3_PMU_EVT_DLY_WR_PAY_M1_TIME 0x0021a216#define HNS3_PMU_EVT_DLY_WR_PAY_M1_PACKET_NUM 0x1021a217#define HNS3_PMU_EVT_DLY_MSIX_WRITE_TIME 0x0021c218#define HNS3_PMU_EVT_DLY_MSIX_WRITE_PACKET_NUM 0x1021c219 220/* interrupt rate events */221#define HNS3_PMU_EVT_PPS_MSIX_NIC_INTR_NUM 0x00300222#define HNS3_PMU_EVT_PPS_MSIX_NIC_TIME 0x10300223 224/* filter mode supported by each bandwidth event */225#define HNS3_PMU_FILTER_BW_SSU_EGU 0x07226#define HNS3_PMU_FILTER_BW_SSU_RPU 0x1f227#define HNS3_PMU_FILTER_BW_SSU_ROCE 0x0f228#define HNS3_PMU_FILTER_BW_ROCE_SSU 0x0f229#define HNS3_PMU_FILTER_BW_TPU_SSU 0x1f230#define HNS3_PMU_FILTER_BW_RPU_RCBRX 0x11231#define HNS3_PMU_FILTER_BW_RCBTX_TXSCH 0x11232#define HNS3_PMU_FILTER_BW_WR_FBD 0x1b233#define HNS3_PMU_FILTER_BW_WR_EBD 0x11234#define HNS3_PMU_FILTER_BW_RD_FBD 0x01235#define HNS3_PMU_FILTER_BW_RD_EBD 0x1b236#define HNS3_PMU_FILTER_BW_RD_PAY_M0 0x01237#define HNS3_PMU_FILTER_BW_RD_PAY_M1 0x01238#define HNS3_PMU_FILTER_BW_WR_PAY_M0 0x01239#define HNS3_PMU_FILTER_BW_WR_PAY_M1 0x01240 241/* filter mode supported by each packet rate event */242#define HNS3_PMU_FILTER_PPS_IGU_SSU 0x07243#define HNS3_PMU_FILTER_PPS_SSU_EGU 0x07244#define HNS3_PMU_FILTER_PPS_SSU_RPU 0x1f245#define HNS3_PMU_FILTER_PPS_SSU_ROCE 0x0f246#define HNS3_PMU_FILTER_PPS_ROCE_SSU 0x0f247#define HNS3_PMU_FILTER_PPS_TPU_SSU 0x1f248#define HNS3_PMU_FILTER_PPS_RPU_RCBRX 0x11249#define HNS3_PMU_FILTER_PPS_RCBTX_TPU 0x1f250#define HNS3_PMU_FILTER_PPS_RCBTX_TXSCH 0x11251#define HNS3_PMU_FILTER_PPS_WR_FBD 0x1b252#define HNS3_PMU_FILTER_PPS_WR_EBD 0x11253#define HNS3_PMU_FILTER_PPS_RD_FBD 0x01254#define HNS3_PMU_FILTER_PPS_RD_EBD 0x1b255#define HNS3_PMU_FILTER_PPS_RD_PAY_M0 0x01256#define HNS3_PMU_FILTER_PPS_RD_PAY_M1 0x01257#define HNS3_PMU_FILTER_PPS_WR_PAY_M0 0x01258#define HNS3_PMU_FILTER_PPS_WR_PAY_M1 0x01259#define HNS3_PMU_FILTER_PPS_NICROH_TX_PRE 0x01260#define HNS3_PMU_FILTER_PPS_NICROH_RX_PRE 0x01261 262/* filter mode supported by each latency event */263#define HNS3_PMU_FILTER_DLY_TX_PUSH 0x01264#define HNS3_PMU_FILTER_DLY_TX 0x01265#define HNS3_PMU_FILTER_DLY_SSU_TX_NIC 0x07266#define HNS3_PMU_FILTER_DLY_SSU_TX_ROCE 0x07267#define HNS3_PMU_FILTER_DLY_SSU_RX_NIC 0x07268#define HNS3_PMU_FILTER_DLY_SSU_RX_ROCE 0x07269#define HNS3_PMU_FILTER_DLY_RPU 0x11270#define HNS3_PMU_FILTER_DLY_TPU 0x1f271#define HNS3_PMU_FILTER_DLY_RPE 0x01272#define HNS3_PMU_FILTER_DLY_TPE 0x0b273#define HNS3_PMU_FILTER_DLY_TPE_PUSH 0x1b274#define HNS3_PMU_FILTER_DLY_WR_FBD 0x1b275#define HNS3_PMU_FILTER_DLY_WR_EBD 0x11276#define HNS3_PMU_FILTER_DLY_RD_FBD 0x01277#define HNS3_PMU_FILTER_DLY_RD_EBD 0x1b278#define HNS3_PMU_FILTER_DLY_RD_PAY_M0 0x01279#define HNS3_PMU_FILTER_DLY_RD_PAY_M1 0x01280#define HNS3_PMU_FILTER_DLY_WR_PAY_M0 0x01281#define HNS3_PMU_FILTER_DLY_WR_PAY_M1 0x01282#define HNS3_PMU_FILTER_DLY_MSIX_WRITE 0x01283 284/* filter mode supported by each interrupt rate event */285#define HNS3_PMU_FILTER_INTR_MSIX_NIC 0x01286 287enum hns3_pmu_hw_filter_mode {288 HNS3_PMU_HW_FILTER_GLOBAL,289 HNS3_PMU_HW_FILTER_PORT,290 HNS3_PMU_HW_FILTER_PORT_TC,291 HNS3_PMU_HW_FILTER_FUNC,292 HNS3_PMU_HW_FILTER_FUNC_QUEUE,293 HNS3_PMU_HW_FILTER_FUNC_INTR,294};295 296struct hns3_pmu_event_attr {297 u32 event;298 u16 filter_support;299};300 301struct hns3_pmu {302 struct perf_event *hw_events[HNS3_PMU_MAX_HW_EVENTS];303 struct hlist_node node;304 struct pci_dev *pdev;305 struct pmu pmu;306 void __iomem *base;307 int irq;308 int on_cpu;309 u32 identifier;310 u32 hw_clk_freq; /* hardware clock frequency of PMU */311 /* maximum and minimum bdf allowed by PMU */312 u16 bdf_min;313 u16 bdf_max;314};315 316#define to_hns3_pmu(p) (container_of((p), struct hns3_pmu, pmu))317 318#define GET_PCI_DEVFN(bdf) ((bdf) & 0xff)319 320#define FILTER_CONDITION_PORT(port) ((1 << (port)) & 0xff)321#define FILTER_CONDITION_PORT_TC(port, tc) (((port) << 3) | ((tc) & 0x07))322#define FILTER_CONDITION_FUNC_INTR(func, intr) (((intr) << 8) | (func))323 324#define HNS3_PMU_FILTER_ATTR(_name, _config, _start, _end) \325 static inline u64 hns3_pmu_get_##_name(struct perf_event *event) \326 { \327 return FIELD_GET(GENMASK_ULL(_end, _start), \328 event->attr._config); \329 }330 331HNS3_PMU_FILTER_ATTR(subevent, config, 0, 7);332HNS3_PMU_FILTER_ATTR(event_type, config, 8, 15);333HNS3_PMU_FILTER_ATTR(ext_counter_used, config, 16, 16);334HNS3_PMU_FILTER_ATTR(port, config1, 0, 3);335HNS3_PMU_FILTER_ATTR(tc, config1, 4, 7);336HNS3_PMU_FILTER_ATTR(bdf, config1, 8, 23);337HNS3_PMU_FILTER_ATTR(queue, config1, 24, 39);338HNS3_PMU_FILTER_ATTR(intr, config1, 40, 51);339HNS3_PMU_FILTER_ATTR(global, config1, 52, 52);340 341#define HNS3_BW_EVT_BYTE_NUM(_name) (&(struct hns3_pmu_event_attr) {\342 HNS3_PMU_EVT_BW_##_name##_BYTE_NUM, \343 HNS3_PMU_FILTER_BW_##_name})344#define HNS3_BW_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\345 HNS3_PMU_EVT_BW_##_name##_TIME, \346 HNS3_PMU_FILTER_BW_##_name})347#define HNS3_PPS_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\348 HNS3_PMU_EVT_PPS_##_name##_PACKET_NUM, \349 HNS3_PMU_FILTER_PPS_##_name})350#define HNS3_PPS_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\351 HNS3_PMU_EVT_PPS_##_name##_TIME, \352 HNS3_PMU_FILTER_PPS_##_name})353#define HNS3_DLY_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\354 HNS3_PMU_EVT_DLY_##_name##_TIME, \355 HNS3_PMU_FILTER_DLY_##_name})356#define HNS3_DLY_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\357 HNS3_PMU_EVT_DLY_##_name##_PACKET_NUM, \358 HNS3_PMU_FILTER_DLY_##_name})359#define HNS3_INTR_EVT_INTR_NUM(_name) (&(struct hns3_pmu_event_attr) {\360 HNS3_PMU_EVT_PPS_##_name##_INTR_NUM, \361 HNS3_PMU_FILTER_INTR_##_name})362#define HNS3_INTR_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\363 HNS3_PMU_EVT_PPS_##_name##_TIME, \364 HNS3_PMU_FILTER_INTR_##_name})365 366static ssize_t hns3_pmu_event_show(struct device *dev,367 struct device_attribute *attr, char *buf)368{369 struct hns3_pmu_event_attr *event;370 struct dev_ext_attribute *eattr;371 372 eattr = container_of(attr, struct dev_ext_attribute, attr);373 event = eattr->var;374 375 return sysfs_emit(buf, "config=0x%x\n", event->event);376}377 378static ssize_t hns3_pmu_filter_mode_show(struct device *dev,379 struct device_attribute *attr,380 char *buf)381{382 struct hns3_pmu_event_attr *event;383 struct dev_ext_attribute *eattr;384 int len;385 386 eattr = container_of(attr, struct dev_ext_attribute, attr);387 event = eattr->var;388 389 len = sysfs_emit_at(buf, 0, "filter mode supported: ");390 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL)391 len += sysfs_emit_at(buf, len, "global ");392 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT)393 len += sysfs_emit_at(buf, len, "port ");394 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC)395 len += sysfs_emit_at(buf, len, "port-tc ");396 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC)397 len += sysfs_emit_at(buf, len, "func ");398 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE)399 len += sysfs_emit_at(buf, len, "func-queue ");400 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR)401 len += sysfs_emit_at(buf, len, "func-intr ");402 403 len += sysfs_emit_at(buf, len, "\n");404 405 return len;406}407 408#define HNS3_PMU_ATTR(_name, _func, _config) \409 (&((struct dev_ext_attribute[]) { \410 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \411 })[0].attr.attr)412 413#define HNS3_PMU_FORMAT_ATTR(_name, _format) \414 HNS3_PMU_ATTR(_name, device_show_string, _format)415#define HNS3_PMU_EVENT_ATTR(_name, _event) \416 HNS3_PMU_ATTR(_name, hns3_pmu_event_show, (void *)_event)417#define HNS3_PMU_FLT_MODE_ATTR(_name, _event) \418 HNS3_PMU_ATTR(_name, hns3_pmu_filter_mode_show, (void *)_event)419 420#define HNS3_PMU_BW_EVT_PAIR(_name, _macro) \421 HNS3_PMU_EVENT_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \422 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))423#define HNS3_PMU_PPS_EVT_PAIR(_name, _macro) \424 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \425 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))426#define HNS3_PMU_DLY_EVT_PAIR(_name, _macro) \427 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \428 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))429#define HNS3_PMU_INTR_EVT_PAIR(_name, _macro) \430 HNS3_PMU_EVENT_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \431 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))432 433#define HNS3_PMU_BW_FLT_MODE_PAIR(_name, _macro) \434 HNS3_PMU_FLT_MODE_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \435 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))436#define HNS3_PMU_PPS_FLT_MODE_PAIR(_name, _macro) \437 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \438 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))439#define HNS3_PMU_DLY_FLT_MODE_PAIR(_name, _macro) \440 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \441 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))442#define HNS3_PMU_INTR_FLT_MODE_PAIR(_name, _macro) \443 HNS3_PMU_FLT_MODE_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \444 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))445 446static u8 hns3_pmu_hw_filter_modes[] = {447 HNS3_PMU_HW_FILTER_GLOBAL,448 HNS3_PMU_HW_FILTER_PORT,449 HNS3_PMU_HW_FILTER_PORT_TC,450 HNS3_PMU_HW_FILTER_FUNC,451 HNS3_PMU_HW_FILTER_FUNC_QUEUE,452 HNS3_PMU_HW_FILTER_FUNC_INTR,453};454 455#define HNS3_PMU_SET_HW_FILTER(_hwc, _mode) \456 ((_hwc)->addr_filters = (void *)&hns3_pmu_hw_filter_modes[(_mode)])457 458static ssize_t identifier_show(struct device *dev,459 struct device_attribute *attr, char *buf)460{461 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));462 463 return sysfs_emit(buf, "0x%x\n", hns3_pmu->identifier);464}465static DEVICE_ATTR_RO(identifier);466 467static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,468 char *buf)469{470 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));471 472 return sysfs_emit(buf, "%d\n", hns3_pmu->on_cpu);473}474static DEVICE_ATTR_RO(cpumask);475 476static ssize_t bdf_min_show(struct device *dev, struct device_attribute *attr,477 char *buf)478{479 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));480 u16 bdf = hns3_pmu->bdf_min;481 482 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),483 PCI_SLOT(bdf), PCI_FUNC(bdf));484}485static DEVICE_ATTR_RO(bdf_min);486 487static ssize_t bdf_max_show(struct device *dev, struct device_attribute *attr,488 char *buf)489{490 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));491 u16 bdf = hns3_pmu->bdf_max;492 493 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),494 PCI_SLOT(bdf), PCI_FUNC(bdf));495}496static DEVICE_ATTR_RO(bdf_max);497 498static ssize_t hw_clk_freq_show(struct device *dev,499 struct device_attribute *attr, char *buf)500{501 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));502 503 return sysfs_emit(buf, "%u\n", hns3_pmu->hw_clk_freq);504}505static DEVICE_ATTR_RO(hw_clk_freq);506 507static struct attribute *hns3_pmu_events_attr[] = {508 /* bandwidth events */509 HNS3_PMU_BW_EVT_PAIR(bw_ssu_egu, SSU_EGU),510 HNS3_PMU_BW_EVT_PAIR(bw_ssu_rpu, SSU_RPU),511 HNS3_PMU_BW_EVT_PAIR(bw_ssu_roce, SSU_ROCE),512 HNS3_PMU_BW_EVT_PAIR(bw_roce_ssu, ROCE_SSU),513 HNS3_PMU_BW_EVT_PAIR(bw_tpu_ssu, TPU_SSU),514 HNS3_PMU_BW_EVT_PAIR(bw_rpu_rcbrx, RPU_RCBRX),515 HNS3_PMU_BW_EVT_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),516 HNS3_PMU_BW_EVT_PAIR(bw_wr_fbd, WR_FBD),517 HNS3_PMU_BW_EVT_PAIR(bw_wr_ebd, WR_EBD),518 HNS3_PMU_BW_EVT_PAIR(bw_rd_fbd, RD_FBD),519 HNS3_PMU_BW_EVT_PAIR(bw_rd_ebd, RD_EBD),520 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m0, RD_PAY_M0),521 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m1, RD_PAY_M1),522 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m0, WR_PAY_M0),523 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m1, WR_PAY_M1),524 525 /* packet rate events */526 HNS3_PMU_PPS_EVT_PAIR(pps_igu_ssu, IGU_SSU),527 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_egu, SSU_EGU),528 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_rpu, SSU_RPU),529 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_roce, SSU_ROCE),530 HNS3_PMU_PPS_EVT_PAIR(pps_roce_ssu, ROCE_SSU),531 HNS3_PMU_PPS_EVT_PAIR(pps_tpu_ssu, TPU_SSU),532 HNS3_PMU_PPS_EVT_PAIR(pps_rpu_rcbrx, RPU_RCBRX),533 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_tpu, RCBTX_TPU),534 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),535 HNS3_PMU_PPS_EVT_PAIR(pps_wr_fbd, WR_FBD),536 HNS3_PMU_PPS_EVT_PAIR(pps_wr_ebd, WR_EBD),537 HNS3_PMU_PPS_EVT_PAIR(pps_rd_fbd, RD_FBD),538 HNS3_PMU_PPS_EVT_PAIR(pps_rd_ebd, RD_EBD),539 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m0, RD_PAY_M0),540 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m1, RD_PAY_M1),541 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m0, WR_PAY_M0),542 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m1, WR_PAY_M1),543 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),544 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),545 546 /* latency events */547 HNS3_PMU_DLY_EVT_PAIR(dly_tx_push_to_mac, TX_PUSH),548 HNS3_PMU_DLY_EVT_PAIR(dly_tx_normal_to_mac, TX),549 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),550 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),551 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),552 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),553 HNS3_PMU_DLY_EVT_PAIR(dly_rpu, RPU),554 HNS3_PMU_DLY_EVT_PAIR(dly_tpu, TPU),555 HNS3_PMU_DLY_EVT_PAIR(dly_rpe, RPE),556 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_normal, TPE),557 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_push, TPE_PUSH),558 HNS3_PMU_DLY_EVT_PAIR(dly_wr_fbd, WR_FBD),559 HNS3_PMU_DLY_EVT_PAIR(dly_wr_ebd, WR_EBD),560 HNS3_PMU_DLY_EVT_PAIR(dly_rd_fbd, RD_FBD),561 HNS3_PMU_DLY_EVT_PAIR(dly_rd_ebd, RD_EBD),562 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m0, RD_PAY_M0),563 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m1, RD_PAY_M1),564 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m0, WR_PAY_M0),565 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m1, WR_PAY_M1),566 HNS3_PMU_DLY_EVT_PAIR(dly_msix_write, MSIX_WRITE),567 568 /* interrupt rate events */569 HNS3_PMU_INTR_EVT_PAIR(pps_intr_msix_nic, MSIX_NIC),570 571 NULL572};573 574static struct attribute *hns3_pmu_filter_mode_attr[] = {575 /* bandwidth events */576 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_egu, SSU_EGU),577 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_rpu, SSU_RPU),578 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_roce, SSU_ROCE),579 HNS3_PMU_BW_FLT_MODE_PAIR(bw_roce_ssu, ROCE_SSU),580 HNS3_PMU_BW_FLT_MODE_PAIR(bw_tpu_ssu, TPU_SSU),581 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rpu_rcbrx, RPU_RCBRX),582 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),583 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_fbd, WR_FBD),584 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_ebd, WR_EBD),585 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_fbd, RD_FBD),586 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_ebd, RD_EBD),587 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m0, RD_PAY_M0),588 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m1, RD_PAY_M1),589 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m0, WR_PAY_M0),590 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m1, WR_PAY_M1),591 592 /* packet rate events */593 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_igu_ssu, IGU_SSU),594 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_egu, SSU_EGU),595 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_rpu, SSU_RPU),596 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_roce, SSU_ROCE),597 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_roce_ssu, ROCE_SSU),598 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_tpu_ssu, TPU_SSU),599 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rpu_rcbrx, RPU_RCBRX),600 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_tpu, RCBTX_TPU),601 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),602 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_fbd, WR_FBD),603 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_ebd, WR_EBD),604 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_fbd, RD_FBD),605 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_ebd, RD_EBD),606 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m0, RD_PAY_M0),607 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m1, RD_PAY_M1),608 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m0, WR_PAY_M0),609 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m1, WR_PAY_M1),610 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),611 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),612 613 /* latency events */614 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_push_to_mac, TX_PUSH),615 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_normal_to_mac, TX),616 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),617 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),618 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),619 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),620 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpu, RPU),621 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpu, TPU),622 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpe, RPE),623 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_normal, TPE),624 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_push, TPE_PUSH),625 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_fbd, WR_FBD),626 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_ebd, WR_EBD),627 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_fbd, RD_FBD),628 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_ebd, RD_EBD),629 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m0, RD_PAY_M0),630 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m1, RD_PAY_M1),631 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m0, WR_PAY_M0),632 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m1, WR_PAY_M1),633 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_msix_write, MSIX_WRITE),634 635 /* interrupt rate events */636 HNS3_PMU_INTR_FLT_MODE_PAIR(pps_intr_msix_nic, MSIX_NIC),637 638 NULL639};640 641static struct attribute_group hns3_pmu_events_group = {642 .name = "events",643 .attrs = hns3_pmu_events_attr,644};645 646static struct attribute_group hns3_pmu_filter_mode_group = {647 .name = "filtermode",648 .attrs = hns3_pmu_filter_mode_attr,649};650 651static struct attribute *hns3_pmu_format_attr[] = {652 HNS3_PMU_FORMAT_ATTR(subevent, "config:0-7"),653 HNS3_PMU_FORMAT_ATTR(event_type, "config:8-15"),654 HNS3_PMU_FORMAT_ATTR(ext_counter_used, "config:16"),655 HNS3_PMU_FORMAT_ATTR(port, "config1:0-3"),656 HNS3_PMU_FORMAT_ATTR(tc, "config1:4-7"),657 HNS3_PMU_FORMAT_ATTR(bdf, "config1:8-23"),658 HNS3_PMU_FORMAT_ATTR(queue, "config1:24-39"),659 HNS3_PMU_FORMAT_ATTR(intr, "config1:40-51"),660 HNS3_PMU_FORMAT_ATTR(global, "config1:52"),661 NULL662};663 664static struct attribute_group hns3_pmu_format_group = {665 .name = "format",666 .attrs = hns3_pmu_format_attr,667};668 669static struct attribute *hns3_pmu_cpumask_attrs[] = {670 &dev_attr_cpumask.attr,671 NULL672};673 674static struct attribute_group hns3_pmu_cpumask_attr_group = {675 .attrs = hns3_pmu_cpumask_attrs,676};677 678static struct attribute *hns3_pmu_identifier_attrs[] = {679 &dev_attr_identifier.attr,680 NULL681};682 683static struct attribute_group hns3_pmu_identifier_attr_group = {684 .attrs = hns3_pmu_identifier_attrs,685};686 687static struct attribute *hns3_pmu_bdf_range_attrs[] = {688 &dev_attr_bdf_min.attr,689 &dev_attr_bdf_max.attr,690 NULL691};692 693static struct attribute_group hns3_pmu_bdf_range_attr_group = {694 .attrs = hns3_pmu_bdf_range_attrs,695};696 697static struct attribute *hns3_pmu_hw_clk_freq_attrs[] = {698 &dev_attr_hw_clk_freq.attr,699 NULL700};701 702static struct attribute_group hns3_pmu_hw_clk_freq_attr_group = {703 .attrs = hns3_pmu_hw_clk_freq_attrs,704};705 706static const struct attribute_group *hns3_pmu_attr_groups[] = {707 &hns3_pmu_events_group,708 &hns3_pmu_filter_mode_group,709 &hns3_pmu_format_group,710 &hns3_pmu_cpumask_attr_group,711 &hns3_pmu_identifier_attr_group,712 &hns3_pmu_bdf_range_attr_group,713 &hns3_pmu_hw_clk_freq_attr_group,714 NULL715};716 717static u32 hns3_pmu_get_event(struct perf_event *event)718{719 return hns3_pmu_get_ext_counter_used(event) << 16 |720 hns3_pmu_get_event_type(event) << 8 |721 hns3_pmu_get_subevent(event);722}723 724static u32 hns3_pmu_get_real_event(struct perf_event *event)725{726 return hns3_pmu_get_event_type(event) << 8 |727 hns3_pmu_get_subevent(event);728}729 730static u32 hns3_pmu_get_offset(u32 offset, u32 idx)731{732 return offset + HNS3_PMU_REG_EVENT_OFFSET +733 HNS3_PMU_REG_EVENT_SIZE * idx;734}735 736static u32 hns3_pmu_readl(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)737{738 u32 offset = hns3_pmu_get_offset(reg_offset, idx);739 740 return readl(hns3_pmu->base + offset);741}742 743static void hns3_pmu_writel(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,744 u32 val)745{746 u32 offset = hns3_pmu_get_offset(reg_offset, idx);747 748 writel(val, hns3_pmu->base + offset);749}750 751static u64 hns3_pmu_readq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)752{753 u32 offset = hns3_pmu_get_offset(reg_offset, idx);754 755 return readq(hns3_pmu->base + offset);756}757 758static void hns3_pmu_writeq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,759 u64 val)760{761 u32 offset = hns3_pmu_get_offset(reg_offset, idx);762 763 writeq(val, hns3_pmu->base + offset);764}765 766static bool hns3_pmu_cmp_event(struct perf_event *target,767 struct perf_event *event)768{769 return hns3_pmu_get_real_event(target) == hns3_pmu_get_real_event(event);770}771 772static int hns3_pmu_find_related_event_idx(struct hns3_pmu *hns3_pmu,773 struct perf_event *event)774{775 struct perf_event *sibling;776 int hw_event_used = 0;777 int idx;778 779 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {780 sibling = hns3_pmu->hw_events[idx];781 if (!sibling)782 continue;783 784 hw_event_used++;785 786 if (!hns3_pmu_cmp_event(sibling, event))787 continue;788 789 /* Related events is used in group */790 if (sibling->group_leader == event->group_leader)791 return idx;792 }793 794 /* No related event and all hardware events are used up */795 if (hw_event_used >= HNS3_PMU_MAX_HW_EVENTS)796 return -EBUSY;797 798 /* No related event and there is extra hardware events can be use */799 return -ENOENT;800}801 802static int hns3_pmu_get_event_idx(struct hns3_pmu *hns3_pmu)803{804 int idx;805 806 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {807 if (!hns3_pmu->hw_events[idx])808 return idx;809 }810 811 return -EBUSY;812}813 814static bool hns3_pmu_valid_bdf(struct hns3_pmu *hns3_pmu, u16 bdf)815{816 struct pci_dev *pdev;817 818 if (bdf < hns3_pmu->bdf_min || bdf > hns3_pmu->bdf_max) {819 pci_err(hns3_pmu->pdev, "Invalid EP device: %#x!\n", bdf);820 return false;821 }822 823 pdev = pci_get_domain_bus_and_slot(pci_domain_nr(hns3_pmu->pdev->bus),824 PCI_BUS_NUM(bdf),825 GET_PCI_DEVFN(bdf));826 if (!pdev) {827 pci_err(hns3_pmu->pdev, "Nonexistent EP device: %#x!\n", bdf);828 return false;829 }830 831 pci_dev_put(pdev);832 return true;833}834 835static void hns3_pmu_set_qid_para(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,836 u16 queue)837{838 u32 val;839 840 val = GET_PCI_DEVFN(bdf);841 val |= (u32)queue << HNS3_PMU_QID_PARA_QUEUE_S;842 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_PARA, idx, val);843}844 845static bool hns3_pmu_qid_req_start(struct hns3_pmu *hns3_pmu, u32 idx)846{847 bool queue_id_valid = false;848 u32 reg_qid_ctrl, val;849 int err;850 851 /* enable queue id request */852 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx,853 HNS3_PMU_QID_CTRL_REQ_ENABLE);854 855 reg_qid_ctrl = hns3_pmu_get_offset(HNS3_PMU_REG_EVENT_QID_CTRL, idx);856 err = readl_poll_timeout(hns3_pmu->base + reg_qid_ctrl, val,857 val & HNS3_PMU_QID_CTRL_DONE, 1, 1000);858 if (err == -ETIMEDOUT) {859 pci_err(hns3_pmu->pdev, "QID request timeout!\n");860 goto out;861 }862 863 queue_id_valid = !(val & HNS3_PMU_QID_CTRL_MISS);864 865out:866 /* disable qid request and clear status */867 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 0);868 869 return queue_id_valid;870}871 872static bool hns3_pmu_valid_queue(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,873 u16 queue)874{875 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue);876 877 return hns3_pmu_qid_req_start(hns3_pmu, idx);878}879 880static struct hns3_pmu_event_attr *hns3_pmu_get_pmu_event(u32 event)881{882 struct hns3_pmu_event_attr *pmu_event;883 struct dev_ext_attribute *eattr;884 struct device_attribute *dattr;885 struct attribute *attr;886 u32 i;887 888 for (i = 0; i < ARRAY_SIZE(hns3_pmu_events_attr) - 1; i++) {889 attr = hns3_pmu_events_attr[i];890 dattr = container_of(attr, struct device_attribute, attr);891 eattr = container_of(dattr, struct dev_ext_attribute, attr);892 pmu_event = eattr->var;893 894 if (event == pmu_event->event)895 return pmu_event;896 }897 898 return NULL;899}900 901static int hns3_pmu_set_func_mode(struct perf_event *event,902 struct hns3_pmu *hns3_pmu)903{904 struct hw_perf_event *hwc = &event->hw;905 u16 bdf = hns3_pmu_get_bdf(event);906 907 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))908 return -ENOENT;909 910 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC);911 912 return 0;913}914 915static int hns3_pmu_set_func_queue_mode(struct perf_event *event,916 struct hns3_pmu *hns3_pmu)917{918 u16 queue_id = hns3_pmu_get_queue(event);919 struct hw_perf_event *hwc = &event->hw;920 u16 bdf = hns3_pmu_get_bdf(event);921 922 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))923 return -ENOENT;924 925 if (!hns3_pmu_valid_queue(hns3_pmu, hwc->idx, bdf, queue_id)) {926 pci_err(hns3_pmu->pdev, "Invalid queue: %u\n", queue_id);927 return -ENOENT;928 }929 930 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_QUEUE);931 932 return 0;933}934 935static bool936hns3_pmu_is_enabled_global_mode(struct perf_event *event,937 struct hns3_pmu_event_attr *pmu_event)938{939 u8 global = hns3_pmu_get_global(event);940 941 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL))942 return false;943 944 return global;945}946 947static bool hns3_pmu_is_enabled_func_mode(struct perf_event *event,948 struct hns3_pmu_event_attr *pmu_event)949{950 u16 queue_id = hns3_pmu_get_queue(event);951 u16 bdf = hns3_pmu_get_bdf(event);952 953 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC))954 return false;955 else if (queue_id != HNS3_PMU_FILTER_ALL_QUEUE)956 return false;957 958 return bdf;959}960 961static bool962hns3_pmu_is_enabled_func_queue_mode(struct perf_event *event,963 struct hns3_pmu_event_attr *pmu_event)964{965 u16 queue_id = hns3_pmu_get_queue(event);966 u16 bdf = hns3_pmu_get_bdf(event);967 968 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE))969 return false;970 else if (queue_id == HNS3_PMU_FILTER_ALL_QUEUE)971 return false;972 973 return bdf;974}975 976static bool hns3_pmu_is_enabled_port_mode(struct perf_event *event,977 struct hns3_pmu_event_attr *pmu_event)978{979 u8 tc_id = hns3_pmu_get_tc(event);980 981 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT))982 return false;983 984 return tc_id == HNS3_PMU_FILTER_ALL_TC;985}986 987static bool988hns3_pmu_is_enabled_port_tc_mode(struct perf_event *event,989 struct hns3_pmu_event_attr *pmu_event)990{991 u8 tc_id = hns3_pmu_get_tc(event);992 993 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC))994 return false;995 996 return tc_id != HNS3_PMU_FILTER_ALL_TC;997}998 999static bool1000hns3_pmu_is_enabled_func_intr_mode(struct perf_event *event,1001 struct hns3_pmu *hns3_pmu,1002 struct hns3_pmu_event_attr *pmu_event)1003{1004 u16 bdf = hns3_pmu_get_bdf(event);1005 1006 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR))1007 return false;1008 1009 return hns3_pmu_valid_bdf(hns3_pmu, bdf);1010}1011 1012static int hns3_pmu_select_filter_mode(struct perf_event *event,1013 struct hns3_pmu *hns3_pmu)1014{1015 u32 event_id = hns3_pmu_get_event(event);1016 struct hw_perf_event *hwc = &event->hw;1017 struct hns3_pmu_event_attr *pmu_event;1018 1019 pmu_event = hns3_pmu_get_pmu_event(event_id);1020 if (!pmu_event) {1021 pci_err(hns3_pmu->pdev, "Invalid pmu event\n");1022 return -ENOENT;1023 }1024 1025 if (hns3_pmu_is_enabled_global_mode(event, pmu_event)) {1026 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_GLOBAL);1027 return 0;1028 }1029 1030 if (hns3_pmu_is_enabled_func_mode(event, pmu_event))1031 return hns3_pmu_set_func_mode(event, hns3_pmu);1032 1033 if (hns3_pmu_is_enabled_func_queue_mode(event, pmu_event))1034 return hns3_pmu_set_func_queue_mode(event, hns3_pmu);1035 1036 if (hns3_pmu_is_enabled_port_mode(event, pmu_event)) {1037 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT);1038 return 0;1039 }1040 1041 if (hns3_pmu_is_enabled_port_tc_mode(event, pmu_event)) {1042 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT_TC);1043 return 0;1044 }1045 1046 if (hns3_pmu_is_enabled_func_intr_mode(event, hns3_pmu, pmu_event)) {1047 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_INTR);1048 return 0;1049 }1050 1051 return -ENOENT;1052}1053 1054static bool hns3_pmu_validate_event_group(struct perf_event *event)1055{1056 struct perf_event *sibling, *leader = event->group_leader;1057 struct perf_event *event_group[HNS3_PMU_MAX_HW_EVENTS];1058 int counters = 1;1059 int num;1060 1061 event_group[0] = leader;1062 if (!is_software_event(leader)) {1063 if (leader->pmu != event->pmu)1064 return false;1065 1066 if (leader != event && !hns3_pmu_cmp_event(leader, event))1067 event_group[counters++] = event;1068 }1069 1070 for_each_sibling_event(sibling, event->group_leader) {1071 if (is_software_event(sibling))1072 continue;1073 1074 if (sibling->pmu != event->pmu)1075 return false;1076 1077 for (num = 0; num < counters; num++) {1078 /*1079 * If we find a related event, then it's a valid group1080 * since we don't need to allocate a new counter for it.1081 */1082 if (hns3_pmu_cmp_event(event_group[num], sibling))1083 break;1084 }1085 1086 /*1087 * Otherwise it's a new event but if there's no available counter,1088 * fail the check since we cannot schedule all the events in1089 * the group simultaneously.1090 */1091 if (num == HNS3_PMU_MAX_HW_EVENTS)1092 return false;1093 1094 if (num == counters)1095 event_group[counters++] = sibling;1096 }1097 1098 return true;1099}1100 1101static u32 hns3_pmu_get_filter_condition(struct perf_event *event)1102{1103 struct hw_perf_event *hwc = &event->hw;1104 u16 intr_id = hns3_pmu_get_intr(event);1105 u8 port_id = hns3_pmu_get_port(event);1106 u16 bdf = hns3_pmu_get_bdf(event);1107 u8 tc_id = hns3_pmu_get_tc(event);1108 u8 filter_mode;1109 1110 filter_mode = *(u8 *)hwc->addr_filters;1111 switch (filter_mode) {1112 case HNS3_PMU_HW_FILTER_PORT:1113 return FILTER_CONDITION_PORT(port_id);1114 case HNS3_PMU_HW_FILTER_PORT_TC:1115 return FILTER_CONDITION_PORT_TC(port_id, tc_id);1116 case HNS3_PMU_HW_FILTER_FUNC:1117 case HNS3_PMU_HW_FILTER_FUNC_QUEUE:1118 return GET_PCI_DEVFN(bdf);1119 case HNS3_PMU_HW_FILTER_FUNC_INTR:1120 return FILTER_CONDITION_FUNC_INTR(GET_PCI_DEVFN(bdf), intr_id);1121 default:1122 break;1123 }1124 1125 return 0;1126}1127 1128static void hns3_pmu_config_filter(struct perf_event *event)1129{1130 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1131 u8 event_type = hns3_pmu_get_event_type(event);1132 u8 subevent_id = hns3_pmu_get_subevent(event);1133 u16 queue_id = hns3_pmu_get_queue(event);1134 struct hw_perf_event *hwc = &event->hw;1135 u8 filter_mode = *(u8 *)hwc->addr_filters;1136 u16 bdf = hns3_pmu_get_bdf(event);1137 u32 idx = hwc->idx;1138 u32 val;1139 1140 val = event_type;1141 val |= subevent_id << HNS3_PMU_CTRL_SUBEVENT_S;1142 val |= filter_mode << HNS3_PMU_CTRL_FILTER_MODE_S;1143 val |= HNS3_PMU_EVENT_OVERFLOW_RESTART;1144 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);1145 1146 val = hns3_pmu_get_filter_condition(event);1147 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_HIGH, idx, val);1148 1149 if (filter_mode == HNS3_PMU_HW_FILTER_FUNC_QUEUE)1150 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue_id);1151}1152 1153static void hns3_pmu_enable_counter(struct hns3_pmu *hns3_pmu,1154 struct hw_perf_event *hwc)1155{1156 u32 idx = hwc->idx;1157 u32 val;1158 1159 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);1160 val |= HNS3_PMU_EVENT_EN;1161 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);1162}1163 1164static void hns3_pmu_disable_counter(struct hns3_pmu *hns3_pmu,1165 struct hw_perf_event *hwc)1166{1167 u32 idx = hwc->idx;1168 u32 val;1169 1170 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);1171 val &= ~HNS3_PMU_EVENT_EN;1172 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);1173}1174 1175static void hns3_pmu_enable_intr(struct hns3_pmu *hns3_pmu,1176 struct hw_perf_event *hwc)1177{1178 u32 idx = hwc->idx;1179 u32 val;1180 1181 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);1182 val &= ~HNS3_PMU_INTR_MASK_OVERFLOW;1183 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);1184}1185 1186static void hns3_pmu_disable_intr(struct hns3_pmu *hns3_pmu,1187 struct hw_perf_event *hwc)1188{1189 u32 idx = hwc->idx;1190 u32 val;1191 1192 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);1193 val |= HNS3_PMU_INTR_MASK_OVERFLOW;1194 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);1195}1196 1197static void hns3_pmu_clear_intr_status(struct hns3_pmu *hns3_pmu, u32 idx)1198{1199 u32 val;1200 1201 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);1202 val |= HNS3_PMU_EVENT_STATUS_RESET;1203 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);1204 1205 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);1206 val &= ~HNS3_PMU_EVENT_STATUS_RESET;1207 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);1208}1209 1210static u64 hns3_pmu_read_counter(struct perf_event *event)1211{1212 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1213 1214 return hns3_pmu_readq(hns3_pmu, event->hw.event_base, event->hw.idx);1215}1216 1217static void hns3_pmu_write_counter(struct perf_event *event, u64 value)1218{1219 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1220 u32 idx = event->hw.idx;1221 1222 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_COUNTER, idx, value);1223 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_EXT_COUNTER, idx, value);1224}1225 1226static void hns3_pmu_init_counter(struct perf_event *event)1227{1228 struct hw_perf_event *hwc = &event->hw;1229 1230 local64_set(&hwc->prev_count, 0);1231 hns3_pmu_write_counter(event, 0);1232}1233 1234static int hns3_pmu_event_init(struct perf_event *event)1235{1236 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1237 struct hw_perf_event *hwc = &event->hw;1238 int idx;1239 int ret;1240 1241 if (event->attr.type != event->pmu->type)1242 return -ENOENT;1243 1244 /* Sampling is not supported */1245 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)1246 return -EOPNOTSUPP;1247 1248 event->cpu = hns3_pmu->on_cpu;1249 1250 idx = hns3_pmu_get_event_idx(hns3_pmu);1251 if (idx < 0) {1252 pci_err(hns3_pmu->pdev, "Up to %u events are supported!\n",1253 HNS3_PMU_MAX_HW_EVENTS);1254 return -EBUSY;1255 }1256 1257 hwc->idx = idx;1258 1259 ret = hns3_pmu_select_filter_mode(event, hns3_pmu);1260 if (ret) {1261 pci_err(hns3_pmu->pdev, "Invalid filter, ret = %d.\n", ret);1262 return ret;1263 }1264 1265 if (!hns3_pmu_validate_event_group(event)) {1266 pci_err(hns3_pmu->pdev, "Invalid event group.\n");1267 return -EINVAL;1268 }1269 1270 if (hns3_pmu_get_ext_counter_used(event))1271 hwc->event_base = HNS3_PMU_REG_EVENT_EXT_COUNTER;1272 else1273 hwc->event_base = HNS3_PMU_REG_EVENT_COUNTER;1274 1275 return 0;1276}1277 1278static void hns3_pmu_read(struct perf_event *event)1279{1280 struct hw_perf_event *hwc = &event->hw;1281 u64 new_cnt, prev_cnt, delta;1282 1283 do {1284 prev_cnt = local64_read(&hwc->prev_count);1285 new_cnt = hns3_pmu_read_counter(event);1286 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) !=1287 prev_cnt);1288 1289 delta = new_cnt - prev_cnt;1290 local64_add(delta, &event->count);1291}1292 1293static void hns3_pmu_start(struct perf_event *event, int flags)1294{1295 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1296 struct hw_perf_event *hwc = &event->hw;1297 1298 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))1299 return;1300 1301 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));1302 hwc->state = 0;1303 1304 hns3_pmu_config_filter(event);1305 hns3_pmu_init_counter(event);1306 hns3_pmu_enable_intr(hns3_pmu, hwc);1307 hns3_pmu_enable_counter(hns3_pmu, hwc);1308 1309 perf_event_update_userpage(event);1310}1311 1312static void hns3_pmu_stop(struct perf_event *event, int flags)1313{1314 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1315 struct hw_perf_event *hwc = &event->hw;1316 1317 hns3_pmu_disable_counter(hns3_pmu, hwc);1318 hns3_pmu_disable_intr(hns3_pmu, hwc);1319 1320 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);1321 hwc->state |= PERF_HES_STOPPED;1322 1323 if (hwc->state & PERF_HES_UPTODATE)1324 return;1325 1326 /* Read hardware counter and update the perf counter statistics */1327 hns3_pmu_read(event);1328 hwc->state |= PERF_HES_UPTODATE;1329}1330 1331static int hns3_pmu_add(struct perf_event *event, int flags)1332{1333 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1334 struct hw_perf_event *hwc = &event->hw;1335 int idx;1336 1337 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;1338 1339 /* Check all working events to find a related event. */1340 idx = hns3_pmu_find_related_event_idx(hns3_pmu, event);1341 if (idx < 0 && idx != -ENOENT)1342 return idx;1343 1344 /* Current event shares an enabled hardware event with related event */1345 if (idx >= 0 && idx < HNS3_PMU_MAX_HW_EVENTS) {1346 hwc->idx = idx;1347 goto start_count;1348 }1349 1350 idx = hns3_pmu_get_event_idx(hns3_pmu);1351 if (idx < 0)1352 return idx;1353 1354 hwc->idx = idx;1355 hns3_pmu->hw_events[idx] = event;1356 1357start_count:1358 if (flags & PERF_EF_START)1359 hns3_pmu_start(event, PERF_EF_RELOAD);1360 1361 return 0;1362}1363 1364static void hns3_pmu_del(struct perf_event *event, int flags)1365{1366 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);1367 struct hw_perf_event *hwc = &event->hw;1368 1369 hns3_pmu_stop(event, PERF_EF_UPDATE);1370 hns3_pmu->hw_events[hwc->idx] = NULL;1371 perf_event_update_userpage(event);1372}1373 1374static void hns3_pmu_enable(struct pmu *pmu)1375{1376 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);1377 u32 val;1378 1379 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);1380 val |= HNS3_PMU_GLOBAL_START;1381 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);1382}1383 1384static void hns3_pmu_disable(struct pmu *pmu)1385{1386 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);1387 u32 val;1388 1389 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);1390 val &= ~HNS3_PMU_GLOBAL_START;1391 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);1392}1393 1394static int hns3_pmu_alloc_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)1395{1396 u16 device_id;1397 char *name;1398 u32 val;1399 1400 hns3_pmu->base = pcim_iomap_table(pdev)[BAR_2];1401 if (!hns3_pmu->base) {1402 pci_err(pdev, "ioremap failed\n");1403 return -ENOMEM;1404 }1405 1406 hns3_pmu->hw_clk_freq = readl(hns3_pmu->base + HNS3_PMU_REG_CLOCK_FREQ);1407 1408 val = readl(hns3_pmu->base + HNS3_PMU_REG_BDF);1409 hns3_pmu->bdf_min = val & 0xffff;1410 hns3_pmu->bdf_max = val >> 16;1411 1412 val = readl(hns3_pmu->base + HNS3_PMU_REG_DEVICE_ID);1413 device_id = val & 0xffff;1414 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hns3_pmu_sicl_%u", device_id);1415 if (!name)1416 return -ENOMEM;1417 1418 hns3_pmu->pdev = pdev;1419 hns3_pmu->on_cpu = -1;1420 hns3_pmu->identifier = readl(hns3_pmu->base + HNS3_PMU_REG_VERSION);1421 hns3_pmu->pmu = (struct pmu) {1422 .name = name,1423 .module = THIS_MODULE,1424 .parent = &pdev->dev,1425 .event_init = hns3_pmu_event_init,1426 .pmu_enable = hns3_pmu_enable,1427 .pmu_disable = hns3_pmu_disable,1428 .add = hns3_pmu_add,1429 .del = hns3_pmu_del,1430 .start = hns3_pmu_start,1431 .stop = hns3_pmu_stop,1432 .read = hns3_pmu_read,1433 .task_ctx_nr = perf_invalid_context,1434 .attr_groups = hns3_pmu_attr_groups,1435 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,1436 };1437 1438 return 0;1439}1440 1441static irqreturn_t hns3_pmu_irq(int irq, void *data)1442{1443 struct hns3_pmu *hns3_pmu = data;1444 u32 intr_status, idx;1445 1446 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {1447 intr_status = hns3_pmu_readl(hns3_pmu,1448 HNS3_PMU_REG_EVENT_INTR_STATUS,1449 idx);1450 1451 /*1452 * As each counter will restart from 0 when it is overflowed,1453 * extra processing is no need, just clear interrupt status.1454 */1455 if (intr_status)1456 hns3_pmu_clear_intr_status(hns3_pmu, idx);1457 }1458 1459 return IRQ_HANDLED;1460}1461 1462static int hns3_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)1463{1464 struct hns3_pmu *hns3_pmu;1465 1466 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);1467 if (!hns3_pmu)1468 return -ENODEV;1469 1470 if (hns3_pmu->on_cpu == -1) {1471 hns3_pmu->on_cpu = cpu;1472 irq_set_affinity(hns3_pmu->irq, cpumask_of(cpu));1473 }1474 1475 return 0;1476}1477 1478static int hns3_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)1479{1480 struct hns3_pmu *hns3_pmu;1481 unsigned int target;1482 1483 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);1484 if (!hns3_pmu)1485 return -ENODEV;1486 1487 /* Nothing to do if this CPU doesn't own the PMU */1488 if (hns3_pmu->on_cpu != cpu)1489 return 0;1490 1491 /* Choose a new CPU from all online cpus */1492 target = cpumask_any_but(cpu_online_mask, cpu);1493 if (target >= nr_cpu_ids)1494 return 0;1495 1496 perf_pmu_migrate_context(&hns3_pmu->pmu, cpu, target);1497 hns3_pmu->on_cpu = target;1498 irq_set_affinity(hns3_pmu->irq, cpumask_of(target));1499 1500 return 0;1501}1502 1503static void hns3_pmu_free_irq(void *data)1504{1505 struct pci_dev *pdev = data;1506 1507 pci_free_irq_vectors(pdev);1508}1509 1510static int hns3_pmu_irq_register(struct pci_dev *pdev,1511 struct hns3_pmu *hns3_pmu)1512{1513 int irq, ret;1514 1515 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);1516 if (ret < 0) {1517 pci_err(pdev, "failed to enable MSI vectors, ret = %d.\n", ret);1518 return ret;1519 }1520 1521 ret = devm_add_action_or_reset(&pdev->dev, hns3_pmu_free_irq, pdev);1522 if (ret) {1523 pci_err(pdev, "failed to add free irq action, ret = %d.\n", ret);1524 return ret;1525 }1526 1527 irq = pci_irq_vector(pdev, 0);1528 ret = devm_request_irq(&pdev->dev, irq, hns3_pmu_irq, 0,1529 hns3_pmu->pmu.name, hns3_pmu);1530 if (ret) {1531 pci_err(pdev, "failed to register irq, ret = %d.\n", ret);1532 return ret;1533 }1534 1535 hns3_pmu->irq = irq;1536 1537 return 0;1538}1539 1540static int hns3_pmu_init_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)1541{1542 int ret;1543 1544 ret = hns3_pmu_alloc_pmu(pdev, hns3_pmu);1545 if (ret)1546 return ret;1547 1548 ret = hns3_pmu_irq_register(pdev, hns3_pmu);1549 if (ret)1550 return ret;1551 1552 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,1553 &hns3_pmu->node);1554 if (ret) {1555 pci_err(pdev, "failed to register hotplug, ret = %d.\n", ret);1556 return ret;1557 }1558 1559 ret = perf_pmu_register(&hns3_pmu->pmu, hns3_pmu->pmu.name, -1);1560 if (ret) {1561 pci_err(pdev, "failed to register perf PMU, ret = %d.\n", ret);1562 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,1563 &hns3_pmu->node);1564 }1565 1566 return ret;1567}1568 1569static void hns3_pmu_uninit_pmu(struct pci_dev *pdev)1570{1571 struct hns3_pmu *hns3_pmu = pci_get_drvdata(pdev);1572 1573 perf_pmu_unregister(&hns3_pmu->pmu);1574 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,1575 &hns3_pmu->node);1576}1577 1578static int hns3_pmu_init_dev(struct pci_dev *pdev)1579{1580 int ret;1581 1582 ret = pcim_enable_device(pdev);1583 if (ret) {1584 pci_err(pdev, "failed to enable pci device, ret = %d.\n", ret);1585 return ret;1586 }1587 1588 ret = pcim_iomap_regions(pdev, BIT(BAR_2), "hns3_pmu");1589 if (ret < 0) {1590 pci_err(pdev, "failed to request pci region, ret = %d.\n", ret);1591 return ret;1592 }1593 1594 pci_set_master(pdev);1595 1596 return 0;1597}1598 1599static int hns3_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id)1600{1601 struct hns3_pmu *hns3_pmu;1602 int ret;1603 1604 hns3_pmu = devm_kzalloc(&pdev->dev, sizeof(*hns3_pmu), GFP_KERNEL);1605 if (!hns3_pmu)1606 return -ENOMEM;1607 1608 ret = hns3_pmu_init_dev(pdev);1609 if (ret)1610 return ret;1611 1612 ret = hns3_pmu_init_pmu(pdev, hns3_pmu);1613 if (ret) {1614 pci_clear_master(pdev);1615 return ret;1616 }1617 1618 pci_set_drvdata(pdev, hns3_pmu);1619 1620 return ret;1621}1622 1623static void hns3_pmu_remove(struct pci_dev *pdev)1624{1625 hns3_pmu_uninit_pmu(pdev);1626 pci_clear_master(pdev);1627 pci_set_drvdata(pdev, NULL);1628}1629 1630static const struct pci_device_id hns3_pmu_ids[] = {1631 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa22b) },1632 { 0, }1633};1634MODULE_DEVICE_TABLE(pci, hns3_pmu_ids);1635 1636static struct pci_driver hns3_pmu_driver = {1637 .name = "hns3_pmu",1638 .id_table = hns3_pmu_ids,1639 .probe = hns3_pmu_probe,1640 .remove = hns3_pmu_remove,1641};1642 1643static int __init hns3_pmu_module_init(void)1644{1645 int ret;1646 1647 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,1648 "AP_PERF_ARM_HNS3_PMU_ONLINE",1649 hns3_pmu_online_cpu,1650 hns3_pmu_offline_cpu);1651 if (ret) {1652 pr_err("failed to setup HNS3 PMU hotplug, ret = %d.\n", ret);1653 return ret;1654 }1655 1656 ret = pci_register_driver(&hns3_pmu_driver);1657 if (ret) {1658 pr_err("failed to register pci driver, ret = %d.\n", ret);1659 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);1660 }1661 1662 return ret;1663}1664module_init(hns3_pmu_module_init);1665 1666static void __exit hns3_pmu_module_exit(void)1667{1668 pci_unregister_driver(&hns3_pmu_driver);1669 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);1670}1671module_exit(hns3_pmu_module_exit);1672 1673MODULE_DESCRIPTION("HNS3 PMU driver");1674MODULE_LICENSE("GPL v2");1675