886 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.4 * Copyright (C) 2021-2022 Linaro Ltd5 * Author: Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>, based on6 * previous work of Thara Gopinath and msm-4.9 downstream sources.7 */8 9#include <linux/err.h>10#include <linux/interconnect.h>11#include <linux/interrupt.h>12#include <linux/io.h>13#include <linux/kernel.h>14#include <linux/module.h>15#include <linux/of.h>16#include <linux/platform_device.h>17#include <linux/pm_opp.h>18#include <linux/regmap.h>19#include <linux/sizes.h>20#define CREATE_TRACE_POINTS21#include "trace_icc-bwmon.h"22 23/*24 * The BWMON samples data throughput within 'sample_ms' time. With three25 * configurable thresholds (Low, Medium and High) gives four windows (called26 * zones) of current bandwidth:27 *28 * Zone 0: byte count < THRES_LO29 * Zone 1: THRES_LO < byte count < THRES_MED30 * Zone 2: THRES_MED < byte count < THRES_HIGH31 * Zone 3: THRES_HIGH < byte count32 *33 * Zones 0 and 2 are not used by this driver.34 */35 36/* Internal sampling clock frequency */37#define HW_TIMER_HZ 1920000038 39#define BWMON_V4_GLOBAL_IRQ_CLEAR 0x10840#define BWMON_V4_GLOBAL_IRQ_ENABLE 0x10c41/*42 * All values here and further are matching regmap fields, so without absolute43 * register offsets.44 */45#define BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE BIT(0)46 47/*48 * Starting with SDM845, the BWMON4 register space has changed a bit:49 * the global registers were jammed into the beginning of the monitor region.50 * To keep the proper offsets, one would have to map <GLOBAL_BASE 0x200> and51 * <GLOBAL_BASE+0x100 0x300>, which is straight up wrong.52 * To facilitate for that, while allowing the older, arguably more proper53 * implementations to work, offset the global registers by -0x100 to avoid54 * having to map half of the global registers twice.55 */56#define BWMON_V4_845_OFFSET 0x10057#define BWMON_V4_GLOBAL_IRQ_CLEAR_845 (BWMON_V4_GLOBAL_IRQ_CLEAR - BWMON_V4_845_OFFSET)58#define BWMON_V4_GLOBAL_IRQ_ENABLE_845 (BWMON_V4_GLOBAL_IRQ_ENABLE - BWMON_V4_845_OFFSET)59 60#define BWMON_V4_IRQ_STATUS 0x10061#define BWMON_V4_IRQ_CLEAR 0x10862 63#define BWMON_V4_IRQ_ENABLE 0x10c64#define BWMON_IRQ_ENABLE_MASK (BIT(1) | BIT(3))65#define BWMON_V5_IRQ_STATUS 0x00066#define BWMON_V5_IRQ_CLEAR 0x00867#define BWMON_V5_IRQ_ENABLE 0x00c68 69#define BWMON_V4_ENABLE 0x2a070#define BWMON_V5_ENABLE 0x01071#define BWMON_ENABLE_ENABLE BIT(0)72 73#define BWMON_V4_CLEAR 0x2a474#define BWMON_V5_CLEAR 0x01475#define BWMON_CLEAR_CLEAR BIT(0)76#define BWMON_CLEAR_CLEAR_ALL BIT(1)77 78#define BWMON_V4_SAMPLE_WINDOW 0x2a879#define BWMON_V5_SAMPLE_WINDOW 0x02080 81#define BWMON_V4_THRESHOLD_HIGH 0x2ac82#define BWMON_V4_THRESHOLD_MED 0x2b083#define BWMON_V4_THRESHOLD_LOW 0x2b484#define BWMON_V5_THRESHOLD_HIGH 0x02485#define BWMON_V5_THRESHOLD_MED 0x02886#define BWMON_V5_THRESHOLD_LOW 0x02c87 88#define BWMON_V4_ZONE_ACTIONS 0x2b889#define BWMON_V5_ZONE_ACTIONS 0x03090/*91 * Actions to perform on some zone 'z' when current zone hits the threshold:92 * Increment counter of zone 'z'93 */94#define BWMON_ZONE_ACTIONS_INCREMENT(z) (0x2 << ((z) * 2))95/* Clear counter of zone 'z' */96#define BWMON_ZONE_ACTIONS_CLEAR(z) (0x1 << ((z) * 2))97 98/* Zone 0 threshold hit: Clear zone count */99#define BWMON_ZONE_ACTIONS_ZONE0 (BWMON_ZONE_ACTIONS_CLEAR(0))100 101/* Zone 1 threshold hit: Increment zone count & clear lower zones */102#define BWMON_ZONE_ACTIONS_ZONE1 (BWMON_ZONE_ACTIONS_INCREMENT(1) | \103 BWMON_ZONE_ACTIONS_CLEAR(0))104 105/* Zone 2 threshold hit: Increment zone count & clear lower zones */106#define BWMON_ZONE_ACTIONS_ZONE2 (BWMON_ZONE_ACTIONS_INCREMENT(2) | \107 BWMON_ZONE_ACTIONS_CLEAR(1) | \108 BWMON_ZONE_ACTIONS_CLEAR(0))109 110/* Zone 3 threshold hit: Increment zone count & clear lower zones */111#define BWMON_ZONE_ACTIONS_ZONE3 (BWMON_ZONE_ACTIONS_INCREMENT(3) | \112 BWMON_ZONE_ACTIONS_CLEAR(2) | \113 BWMON_ZONE_ACTIONS_CLEAR(1) | \114 BWMON_ZONE_ACTIONS_CLEAR(0))115 116/*117 * There is no clear documentation/explanation of BWMON_V4_THRESHOLD_COUNT118 * register. Based on observations, this is number of times one threshold has to119 * be reached, to trigger interrupt in given zone.120 *121 * 0xff are maximum values meant to ignore the zones 0 and 2.122 */123#define BWMON_V4_THRESHOLD_COUNT 0x2bc124#define BWMON_V5_THRESHOLD_COUNT 0x034125#define BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT 0xff126#define BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT 0xff127 128#define BWMON_V4_ZONE_MAX(zone) (0x2e0 + 4 * (zone))129#define BWMON_V5_ZONE_MAX(zone) (0x044 + 4 * (zone))130 131/* Quirks for specific BWMON types */132#define BWMON_HAS_GLOBAL_IRQ BIT(0)133#define BWMON_NEEDS_FORCE_CLEAR BIT(1)134 135enum bwmon_fields {136 /* Global region fields, keep them at the top */137 F_GLOBAL_IRQ_CLEAR,138 F_GLOBAL_IRQ_ENABLE,139 F_NUM_GLOBAL_FIELDS,140 141 /* Monitor region fields */142 F_IRQ_STATUS = F_NUM_GLOBAL_FIELDS,143 F_IRQ_CLEAR,144 F_IRQ_ENABLE,145 F_ENABLE,146 F_CLEAR,147 F_SAMPLE_WINDOW,148 F_THRESHOLD_HIGH,149 F_THRESHOLD_MED,150 F_THRESHOLD_LOW,151 F_ZONE_ACTIONS_ZONE0,152 F_ZONE_ACTIONS_ZONE1,153 F_ZONE_ACTIONS_ZONE2,154 F_ZONE_ACTIONS_ZONE3,155 F_THRESHOLD_COUNT_ZONE0,156 F_THRESHOLD_COUNT_ZONE1,157 F_THRESHOLD_COUNT_ZONE2,158 F_THRESHOLD_COUNT_ZONE3,159 F_ZONE0_MAX,160 F_ZONE1_MAX,161 F_ZONE2_MAX,162 F_ZONE3_MAX,163 164 F_NUM_FIELDS165};166 167struct icc_bwmon_data {168 unsigned int sample_ms;169 unsigned int count_unit_kb; /* kbytes */170 u8 zone1_thres_count;171 u8 zone3_thres_count;172 unsigned int quirks;173 174 const struct regmap_config *regmap_cfg;175 const struct reg_field *regmap_fields;176 177 const struct regmap_config *global_regmap_cfg;178 const struct reg_field *global_regmap_fields;179};180 181struct icc_bwmon {182 struct device *dev;183 const struct icc_bwmon_data *data;184 int irq;185 186 struct regmap_field *regs[F_NUM_FIELDS];187 struct regmap_field *global_regs[F_NUM_GLOBAL_FIELDS];188 189 unsigned int max_bw_kbps;190 unsigned int min_bw_kbps;191 unsigned int target_kbps;192 unsigned int current_kbps;193};194 195/* BWMON v4 */196static const struct reg_field msm8998_bwmon_reg_fields[] = {197 [F_GLOBAL_IRQ_CLEAR] = {},198 [F_GLOBAL_IRQ_ENABLE] = {},199 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7),200 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7),201 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7),202 /* F_ENABLE covers entire register to disable other features */203 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31),204 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1),205 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23),206 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11),207 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11),208 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11),209 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7),210 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15),211 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23),212 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31),213 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7),214 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15),215 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23),216 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31),217 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11),218 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11),219 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11),220 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11),221};222 223static const struct regmap_range msm8998_bwmon_reg_noread_ranges[] = {224 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR),225 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR),226};227 228static const struct regmap_access_table msm8998_bwmon_reg_read_table = {229 .no_ranges = msm8998_bwmon_reg_noread_ranges,230 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_reg_noread_ranges),231};232 233static const struct regmap_range msm8998_bwmon_reg_volatile_ranges[] = {234 regmap_reg_range(BWMON_V4_IRQ_STATUS, BWMON_V4_IRQ_STATUS),235 regmap_reg_range(BWMON_V4_ZONE_MAX(0), BWMON_V4_ZONE_MAX(3)),236};237 238static const struct regmap_access_table msm8998_bwmon_reg_volatile_table = {239 .yes_ranges = msm8998_bwmon_reg_volatile_ranges,240 .n_yes_ranges = ARRAY_SIZE(msm8998_bwmon_reg_volatile_ranges),241};242 243static const struct reg_field msm8998_bwmon_global_reg_fields[] = {244 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0),245 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0),246};247 248static const struct regmap_range msm8998_bwmon_global_reg_noread_ranges[] = {249 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR),250};251 252static const struct regmap_access_table msm8998_bwmon_global_reg_read_table = {253 .no_ranges = msm8998_bwmon_global_reg_noread_ranges,254 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_global_reg_noread_ranges),255};256 257/*258 * Fill the cache for non-readable registers only as rest does not really259 * matter and can be read from the device.260 */261static const struct reg_default msm8998_bwmon_reg_defaults[] = {262 { BWMON_V4_IRQ_CLEAR, 0x0 },263 { BWMON_V4_CLEAR, 0x0 },264};265 266static const struct reg_default msm8998_bwmon_global_reg_defaults[] = {267 { BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 },268};269 270static const struct regmap_config msm8998_bwmon_regmap_cfg = {271 .reg_bits = 32,272 .reg_stride = 4,273 .val_bits = 32,274 /*275 * No concurrent access expected - driver has one interrupt handler,276 * regmap is not shared, no driver or user-space API.277 */278 .disable_locking = true,279 .rd_table = &msm8998_bwmon_reg_read_table,280 .volatile_table = &msm8998_bwmon_reg_volatile_table,281 .reg_defaults = msm8998_bwmon_reg_defaults,282 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_reg_defaults),283 /*284 * Cache is necessary for using regmap fields with non-readable285 * registers.286 */287 .cache_type = REGCACHE_MAPLE,288};289 290static const struct regmap_config msm8998_bwmon_global_regmap_cfg = {291 .reg_bits = 32,292 .reg_stride = 4,293 .val_bits = 32,294 /*295 * No concurrent access expected - driver has one interrupt handler,296 * regmap is not shared, no driver or user-space API.297 */298 .disable_locking = true,299 .rd_table = &msm8998_bwmon_global_reg_read_table,300 .reg_defaults = msm8998_bwmon_global_reg_defaults,301 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_global_reg_defaults),302 /*303 * Cache is necessary for using regmap fields with non-readable304 * registers.305 */306 .cache_type = REGCACHE_MAPLE,307};308 309static const struct reg_field sdm845_cpu_bwmon_reg_fields[] = {310 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0, 0),311 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE_845, 0, 0),312 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7),313 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7),314 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7),315 /* F_ENABLE covers entire register to disable other features */316 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31),317 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1),318 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23),319 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11),320 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11),321 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11),322 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7),323 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15),324 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23),325 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31),326 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7),327 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15),328 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23),329 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31),330 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11),331 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11),332 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11),333 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11),334};335 336static const struct regmap_range sdm845_cpu_bwmon_reg_noread_ranges[] = {337 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR_845, BWMON_V4_GLOBAL_IRQ_CLEAR_845),338 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR),339 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR),340};341 342static const struct regmap_access_table sdm845_cpu_bwmon_reg_read_table = {343 .no_ranges = sdm845_cpu_bwmon_reg_noread_ranges,344 .n_no_ranges = ARRAY_SIZE(sdm845_cpu_bwmon_reg_noread_ranges),345};346 347/*348 * Fill the cache for non-readable registers only as rest does not really349 * matter and can be read from the device.350 */351static const struct reg_default sdm845_cpu_bwmon_reg_defaults[] = {352 { BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0x0 },353 { BWMON_V4_IRQ_CLEAR, 0x0 },354 { BWMON_V4_CLEAR, 0x0 },355};356 357static const struct regmap_config sdm845_cpu_bwmon_regmap_cfg = {358 .reg_bits = 32,359 .reg_stride = 4,360 .val_bits = 32,361 /*362 * No concurrent access expected - driver has one interrupt handler,363 * regmap is not shared, no driver or user-space API.364 */365 .disable_locking = true,366 .rd_table = &sdm845_cpu_bwmon_reg_read_table,367 .volatile_table = &msm8998_bwmon_reg_volatile_table,368 .reg_defaults = sdm845_cpu_bwmon_reg_defaults,369 .num_reg_defaults = ARRAY_SIZE(sdm845_cpu_bwmon_reg_defaults),370 /*371 * Cache is necessary for using regmap fields with non-readable372 * registers.373 */374 .cache_type = REGCACHE_MAPLE,375};376 377/* BWMON v5 */378static const struct reg_field sdm845_llcc_bwmon_reg_fields[] = {379 [F_GLOBAL_IRQ_CLEAR] = {},380 [F_GLOBAL_IRQ_ENABLE] = {},381 [F_IRQ_STATUS] = REG_FIELD(BWMON_V5_IRQ_STATUS, 0, 3),382 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V5_IRQ_CLEAR, 0, 3),383 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V5_IRQ_ENABLE, 0, 3),384 /* F_ENABLE covers entire register to disable other features */385 [F_ENABLE] = REG_FIELD(BWMON_V5_ENABLE, 0, 31),386 [F_CLEAR] = REG_FIELD(BWMON_V5_CLEAR, 0, 1),387 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V5_SAMPLE_WINDOW, 0, 19),388 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V5_THRESHOLD_HIGH, 0, 11),389 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V5_THRESHOLD_MED, 0, 11),390 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V5_THRESHOLD_LOW, 0, 11),391 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 0, 7),392 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 8, 15),393 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 16, 23),394 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 24, 31),395 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 0, 7),396 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 8, 15),397 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 16, 23),398 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 24, 31),399 [F_ZONE0_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(0), 0, 11),400 [F_ZONE1_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(1), 0, 11),401 [F_ZONE2_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(2), 0, 11),402 [F_ZONE3_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(3), 0, 11),403};404 405static const struct regmap_range sdm845_llcc_bwmon_reg_noread_ranges[] = {406 regmap_reg_range(BWMON_V5_IRQ_CLEAR, BWMON_V5_IRQ_CLEAR),407 regmap_reg_range(BWMON_V5_CLEAR, BWMON_V5_CLEAR),408};409 410static const struct regmap_access_table sdm845_llcc_bwmon_reg_read_table = {411 .no_ranges = sdm845_llcc_bwmon_reg_noread_ranges,412 .n_no_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_noread_ranges),413};414 415static const struct regmap_range sdm845_llcc_bwmon_reg_volatile_ranges[] = {416 regmap_reg_range(BWMON_V5_IRQ_STATUS, BWMON_V5_IRQ_STATUS),417 regmap_reg_range(BWMON_V5_ZONE_MAX(0), BWMON_V5_ZONE_MAX(3)),418};419 420static const struct regmap_access_table sdm845_llcc_bwmon_reg_volatile_table = {421 .yes_ranges = sdm845_llcc_bwmon_reg_volatile_ranges,422 .n_yes_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_volatile_ranges),423};424 425/*426 * Fill the cache for non-readable registers only as rest does not really427 * matter and can be read from the device.428 */429static const struct reg_default sdm845_llcc_bwmon_reg_defaults[] = {430 { BWMON_V5_IRQ_CLEAR, 0x0 },431 { BWMON_V5_CLEAR, 0x0 },432};433 434static const struct regmap_config sdm845_llcc_bwmon_regmap_cfg = {435 .reg_bits = 32,436 .reg_stride = 4,437 .val_bits = 32,438 /*439 * No concurrent access expected - driver has one interrupt handler,440 * regmap is not shared, no driver or user-space API.441 */442 .disable_locking = true,443 .rd_table = &sdm845_llcc_bwmon_reg_read_table,444 .volatile_table = &sdm845_llcc_bwmon_reg_volatile_table,445 .reg_defaults = sdm845_llcc_bwmon_reg_defaults,446 .num_reg_defaults = ARRAY_SIZE(sdm845_llcc_bwmon_reg_defaults),447 /*448 * Cache is necessary for using regmap fields with non-readable449 * registers.450 */451 .cache_type = REGCACHE_MAPLE,452};453 454static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all)455{456 unsigned int val = BWMON_CLEAR_CLEAR;457 458 if (clear_all)459 val |= BWMON_CLEAR_CLEAR_ALL;460 /*461 * Clear counters. The order and barriers are462 * important. Quoting downstream Qualcomm msm-4.9 tree:463 *464 * The counter clear and IRQ clear bits are not in the same 4KB465 * region. So, we need to make sure the counter clear is completed466 * before we try to clear the IRQ or do any other counter operations.467 */468 regmap_field_force_write(bwmon->regs[F_CLEAR], val);469 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR)470 regmap_field_force_write(bwmon->regs[F_CLEAR], 0);471}472 473static void bwmon_clear_irq(struct icc_bwmon *bwmon)474{475 struct regmap_field *global_irq_clr;476 477 if (bwmon->data->global_regmap_fields)478 global_irq_clr = bwmon->global_regs[F_GLOBAL_IRQ_CLEAR];479 else480 global_irq_clr = bwmon->regs[F_GLOBAL_IRQ_CLEAR];481 482 /*483 * Clear zone and global interrupts. The order and barriers are484 * important. Quoting downstream Qualcomm msm-4.9 tree:485 *486 * Synchronize the local interrupt clear in mon_irq_clear()487 * with the global interrupt clear here. Otherwise, the CPU488 * may reorder the two writes and clear the global interrupt489 * before the local interrupt, causing the global interrupt490 * to be retriggered by the local interrupt still being high.491 *492 * Similarly, because the global registers are in a different493 * region than the local registers, we need to ensure any register494 * writes to enable the monitor after this call are ordered with the495 * clearing here so that local writes don't happen before the496 * interrupt is cleared.497 */498 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], BWMON_IRQ_ENABLE_MASK);499 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR)500 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], 0);501 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)502 regmap_field_force_write(global_irq_clr,503 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);504}505 506static void bwmon_disable(struct icc_bwmon *bwmon)507{508 struct regmap_field *global_irq_en;509 510 if (bwmon->data->global_regmap_fields)511 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE];512 else513 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE];514 515 /* Disable interrupts. Strict ordering, see bwmon_clear_irq(). */516 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)517 regmap_field_write(global_irq_en, 0x0);518 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], 0x0);519 520 /*521 * Disable bwmon. Must happen before bwmon_clear_irq() to avoid spurious522 * IRQ.523 */524 regmap_field_write(bwmon->regs[F_ENABLE], 0x0);525}526 527static void bwmon_enable(struct icc_bwmon *bwmon, unsigned int irq_enable)528{529 struct regmap_field *global_irq_en;530 531 if (bwmon->data->global_regmap_fields)532 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE];533 else534 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE];535 536 /* Enable interrupts */537 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)538 regmap_field_write(global_irq_en,539 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);540 541 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], irq_enable);542 543 /* Enable bwmon */544 regmap_field_write(bwmon->regs[F_ENABLE], BWMON_ENABLE_ENABLE);545}546 547static unsigned int bwmon_kbps_to_count(struct icc_bwmon *bwmon,548 unsigned int kbps)549{550 return kbps / bwmon->data->count_unit_kb;551}552 553static void bwmon_set_threshold(struct icc_bwmon *bwmon,554 struct regmap_field *reg, unsigned int kbps)555{556 unsigned int thres;557 558 thres = mult_frac(bwmon_kbps_to_count(bwmon, kbps),559 bwmon->data->sample_ms, MSEC_PER_SEC);560 regmap_field_write(reg, thres);561}562 563static void bwmon_start(struct icc_bwmon *bwmon)564{565 const struct icc_bwmon_data *data = bwmon->data;566 u32 bw_low = 0;567 int window;568 569 /* No need to check for errors, as this must have succeeded before. */570 dev_pm_opp_put(dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_low, 0));571 572 bwmon_clear_counters(bwmon, true);573 574 window = mult_frac(bwmon->data->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC);575 /* Maximum sampling window: 0xffffff for v4 and 0xfffff for v5 */576 regmap_field_write(bwmon->regs[F_SAMPLE_WINDOW], window);577 578 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], bw_low);579 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], bw_low);580 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_LOW], 0);581 582 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE0],583 BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT);584 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE1],585 data->zone1_thres_count);586 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE2],587 BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT);588 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE3],589 data->zone3_thres_count);590 591 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE0],592 BWMON_ZONE_ACTIONS_ZONE0);593 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE1],594 BWMON_ZONE_ACTIONS_ZONE1);595 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE2],596 BWMON_ZONE_ACTIONS_ZONE2);597 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE3],598 BWMON_ZONE_ACTIONS_ZONE3);599 600 bwmon_clear_irq(bwmon);601 bwmon_enable(bwmon, BWMON_IRQ_ENABLE_MASK);602}603 604static irqreturn_t bwmon_intr(int irq, void *dev_id)605{606 struct icc_bwmon *bwmon = dev_id;607 unsigned int status, max;608 int zone;609 610 if (regmap_field_read(bwmon->regs[F_IRQ_STATUS], &status))611 return IRQ_NONE;612 613 status &= BWMON_IRQ_ENABLE_MASK;614 if (!status) {615 /*616 * Only zone 1 and zone 3 interrupts are enabled but zone 2617 * threshold could be hit and trigger interrupt even if not618 * enabled.619 * Such spurious interrupt might come with valuable max count or620 * not, so solution would be to always check all621 * BWMON_ZONE_MAX() registers to find the highest value.622 * Such case is currently ignored.623 */624 return IRQ_NONE;625 }626 627 bwmon_disable(bwmon);628 629 zone = get_bitmask_order(status) - 1;630 /*631 * Zone max bytes count register returns count units within sampling632 * window. Downstream kernel for BWMONv4 (called BWMON type 2 in633 * downstream) always increments the max bytes count by one.634 */635 if (regmap_field_read(bwmon->regs[F_ZONE0_MAX + zone], &max))636 return IRQ_NONE;637 638 max += 1;639 max *= bwmon->data->count_unit_kb;640 bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->data->sample_ms);641 642 return IRQ_WAKE_THREAD;643}644 645static irqreturn_t bwmon_intr_thread(int irq, void *dev_id)646{647 struct icc_bwmon *bwmon = dev_id;648 unsigned int irq_enable = 0;649 struct dev_pm_opp *opp, *target_opp;650 unsigned int bw_kbps, up_kbps, down_kbps, meas_kbps;651 652 bw_kbps = bwmon->target_kbps;653 meas_kbps = bwmon->target_kbps;654 655 target_opp = dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_kbps, 0);656 if (IS_ERR(target_opp) && PTR_ERR(target_opp) == -ERANGE)657 target_opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0);658 659 bwmon->target_kbps = bw_kbps;660 661 bw_kbps--;662 opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0);663 if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE)664 down_kbps = bwmon->target_kbps;665 else666 down_kbps = bw_kbps;667 668 up_kbps = bwmon->target_kbps + 1;669 670 if (bwmon->target_kbps >= bwmon->max_bw_kbps)671 irq_enable = BIT(1);672 else if (bwmon->target_kbps <= bwmon->min_bw_kbps)673 irq_enable = BIT(3);674 else675 irq_enable = BWMON_IRQ_ENABLE_MASK;676 677 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH],678 up_kbps);679 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED],680 down_kbps);681 bwmon_clear_counters(bwmon, false);682 bwmon_clear_irq(bwmon);683 bwmon_enable(bwmon, irq_enable);684 685 trace_qcom_bwmon_update(dev_name(bwmon->dev), meas_kbps, up_kbps, down_kbps);686 if (bwmon->target_kbps == bwmon->current_kbps)687 goto out;688 689 dev_pm_opp_set_opp(bwmon->dev, target_opp);690 bwmon->current_kbps = bwmon->target_kbps;691 692out:693 dev_pm_opp_put(target_opp);694 if (!IS_ERR(opp))695 dev_pm_opp_put(opp);696 697 return IRQ_HANDLED;698}699 700static int bwmon_init_regmap(struct platform_device *pdev,701 struct icc_bwmon *bwmon)702{703 struct device *dev = &pdev->dev;704 void __iomem *base;705 struct regmap *map;706 int ret;707 708 /* Map the monitor base */709 base = devm_platform_ioremap_resource(pdev, 0);710 if (IS_ERR(base))711 return dev_err_probe(dev, PTR_ERR(base),712 "failed to map bwmon registers\n");713 714 map = devm_regmap_init_mmio(dev, base, bwmon->data->regmap_cfg);715 if (IS_ERR(map))716 return dev_err_probe(dev, PTR_ERR(map),717 "failed to initialize regmap\n");718 719 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_global_reg_fields) != F_NUM_GLOBAL_FIELDS);720 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_reg_fields) != F_NUM_FIELDS);721 BUILD_BUG_ON(ARRAY_SIZE(sdm845_cpu_bwmon_reg_fields) != F_NUM_FIELDS);722 BUILD_BUG_ON(ARRAY_SIZE(sdm845_llcc_bwmon_reg_fields) != F_NUM_FIELDS);723 724 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->regs,725 bwmon->data->regmap_fields,726 F_NUM_FIELDS);727 if (ret)728 return ret;729 730 if (bwmon->data->global_regmap_cfg) {731 /* Map the global base, if separate */732 base = devm_platform_ioremap_resource(pdev, 1);733 if (IS_ERR(base))734 return dev_err_probe(dev, PTR_ERR(base),735 "failed to map bwmon global registers\n");736 737 map = devm_regmap_init_mmio(dev, base, bwmon->data->global_regmap_cfg);738 if (IS_ERR(map))739 return dev_err_probe(dev, PTR_ERR(map),740 "failed to initialize global regmap\n");741 742 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->global_regs,743 bwmon->data->global_regmap_fields,744 F_NUM_GLOBAL_FIELDS);745 }746 747 return ret;748}749 750static int bwmon_probe(struct platform_device *pdev)751{752 struct device *dev = &pdev->dev;753 struct dev_pm_opp *opp;754 struct icc_bwmon *bwmon;755 int ret;756 757 bwmon = devm_kzalloc(dev, sizeof(*bwmon), GFP_KERNEL);758 if (!bwmon)759 return -ENOMEM;760 761 bwmon->data = of_device_get_match_data(dev);762 763 ret = bwmon_init_regmap(pdev, bwmon);764 if (ret)765 return ret;766 767 bwmon->irq = platform_get_irq(pdev, 0);768 if (bwmon->irq < 0)769 return bwmon->irq;770 771 ret = devm_pm_opp_of_add_table(dev);772 if (ret)773 return dev_err_probe(dev, ret, "failed to add OPP table\n");774 775 bwmon->max_bw_kbps = UINT_MAX;776 opp = dev_pm_opp_find_bw_floor(dev, &bwmon->max_bw_kbps, 0);777 if (IS_ERR(opp))778 return dev_err_probe(dev, PTR_ERR(opp), "failed to find max peak bandwidth\n");779 dev_pm_opp_put(opp);780 781 bwmon->min_bw_kbps = 0;782 opp = dev_pm_opp_find_bw_ceil(dev, &bwmon->min_bw_kbps, 0);783 if (IS_ERR(opp))784 return dev_err_probe(dev, PTR_ERR(opp), "failed to find min peak bandwidth\n");785 dev_pm_opp_put(opp);786 787 bwmon->dev = dev;788 789 bwmon_disable(bwmon);790 791 /*792 * SoCs with multiple cpu-bwmon instances can end up using a shared interrupt793 * line. Using the devm_ variant might result in the IRQ handler being executed794 * after bwmon_disable in bwmon_remove()795 */796 ret = request_threaded_irq(bwmon->irq, bwmon_intr, bwmon_intr_thread,797 IRQF_ONESHOT | IRQF_SHARED, dev_name(dev), bwmon);798 if (ret)799 return dev_err_probe(dev, ret, "failed to request IRQ\n");800 801 platform_set_drvdata(pdev, bwmon);802 bwmon_start(bwmon);803 804 return 0;805}806 807static void bwmon_remove(struct platform_device *pdev)808{809 struct icc_bwmon *bwmon = platform_get_drvdata(pdev);810 811 bwmon_disable(bwmon);812 free_irq(bwmon->irq, bwmon);813}814 815static const struct icc_bwmon_data msm8998_bwmon_data = {816 .sample_ms = 4,817 .count_unit_kb = 1024,818 .zone1_thres_count = 16,819 .zone3_thres_count = 1,820 .quirks = BWMON_HAS_GLOBAL_IRQ,821 .regmap_fields = msm8998_bwmon_reg_fields,822 .regmap_cfg = &msm8998_bwmon_regmap_cfg,823 .global_regmap_fields = msm8998_bwmon_global_reg_fields,824 .global_regmap_cfg = &msm8998_bwmon_global_regmap_cfg,825};826 827static const struct icc_bwmon_data sdm845_cpu_bwmon_data = {828 .sample_ms = 4,829 .count_unit_kb = 64,830 .zone1_thres_count = 16,831 .zone3_thres_count = 1,832 .quirks = BWMON_HAS_GLOBAL_IRQ,833 .regmap_fields = sdm845_cpu_bwmon_reg_fields,834 .regmap_cfg = &sdm845_cpu_bwmon_regmap_cfg,835};836 837static const struct icc_bwmon_data sdm845_llcc_bwmon_data = {838 .sample_ms = 4,839 .count_unit_kb = 1024,840 .zone1_thres_count = 16,841 .zone3_thres_count = 1,842 .regmap_fields = sdm845_llcc_bwmon_reg_fields,843 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg,844};845 846static const struct icc_bwmon_data sc7280_llcc_bwmon_data = {847 .sample_ms = 4,848 .count_unit_kb = 64,849 .zone1_thres_count = 16,850 .zone3_thres_count = 1,851 .quirks = BWMON_NEEDS_FORCE_CLEAR,852 .regmap_fields = sdm845_llcc_bwmon_reg_fields,853 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg,854};855 856static const struct of_device_id bwmon_of_match[] = {857 /* BWMONv4, separate monitor and global register spaces */858 { .compatible = "qcom,msm8998-bwmon", .data = &msm8998_bwmon_data },859 /* BWMONv4, unified register space */860 { .compatible = "qcom,sdm845-bwmon", .data = &sdm845_cpu_bwmon_data },861 /* BWMONv5 */862 { .compatible = "qcom,sdm845-llcc-bwmon", .data = &sdm845_llcc_bwmon_data },863 { .compatible = "qcom,sc7280-llcc-bwmon", .data = &sc7280_llcc_bwmon_data },864 865 /* Compatibles kept for legacy reasons */866 { .compatible = "qcom,sc7280-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },867 { .compatible = "qcom,sc8280xp-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },868 { .compatible = "qcom,sm8550-cpu-bwmon", .data = &sdm845_cpu_bwmon_data },869 {}870};871MODULE_DEVICE_TABLE(of, bwmon_of_match);872 873static struct platform_driver bwmon_driver = {874 .probe = bwmon_probe,875 .remove_new = bwmon_remove,876 .driver = {877 .name = "qcom-bwmon",878 .of_match_table = bwmon_of_match,879 },880};881module_platform_driver(bwmon_driver);882 883MODULE_AUTHOR("Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>");884MODULE_DESCRIPTION("QCOM BWMON driver");885MODULE_LICENSE("GPL");886