352 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Apple SoC CPU cluster performance state driver4 *5 * Copyright The Asahi Linux Contributors6 *7 * Based on scpi-cpufreq.c8 */9 10#include <linux/bitfield.h>11#include <linux/bitops.h>12#include <linux/cpu.h>13#include <linux/cpufreq.h>14#include <linux/cpumask.h>15#include <linux/delay.h>16#include <linux/err.h>17#include <linux/io.h>18#include <linux/iopoll.h>19#include <linux/module.h>20#include <linux/of.h>21#include <linux/of_address.h>22#include <linux/pm_opp.h>23#include <linux/slab.h>24 25#define APPLE_DVFS_CMD 0x2026#define APPLE_DVFS_CMD_BUSY BIT(31)27#define APPLE_DVFS_CMD_SET BIT(25)28#define APPLE_DVFS_CMD_PS2 GENMASK(16, 12)29#define APPLE_DVFS_CMD_PS1 GENMASK(4, 0)30 31/* Same timebase as CPU counter (24MHz) */32#define APPLE_DVFS_LAST_CHG_TIME 0x3833 34/*35 * Apple ran out of bits and had to shift this in T8112...36 */37#define APPLE_DVFS_STATUS 0x5038#define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4)39#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 440#define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0)41#define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5)42#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 543#define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0)44 45/*46 * Div is +1, base clock is 12MHz on existing SoCs.47 * For documentation purposes. We use the OPP table to48 * get the frequency.49 */50#define APPLE_DVFS_PLL_STATUS 0xc051#define APPLE_DVFS_PLL_FACTOR 0xc852#define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16)53#define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0)54 55#define APPLE_DVFS_TRANSITION_TIMEOUT 10056 57struct apple_soc_cpufreq_info {58 u64 max_pstate;59 u64 cur_pstate_mask;60 u64 cur_pstate_shift;61};62 63struct apple_cpu_priv {64 struct device *cpu_dev;65 void __iomem *reg_base;66 const struct apple_soc_cpufreq_info *info;67};68 69static struct cpufreq_driver apple_soc_cpufreq_driver;70 71static const struct apple_soc_cpufreq_info soc_t8103_info = {72 .max_pstate = 15,73 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,74 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,75};76 77static const struct apple_soc_cpufreq_info soc_t8112_info = {78 .max_pstate = 31,79 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,80 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,81};82 83static const struct apple_soc_cpufreq_info soc_default_info = {84 .max_pstate = 15,85 .cur_pstate_mask = 0, /* fallback */86};87 88static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = {89 {90 .compatible = "apple,t8103-cluster-cpufreq",91 .data = &soc_t8103_info,92 },93 {94 .compatible = "apple,t8112-cluster-cpufreq",95 .data = &soc_t8112_info,96 },97 {98 .compatible = "apple,cluster-cpufreq",99 .data = &soc_default_info,100 },101 {}102};103 104static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)105{106 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);107 struct apple_cpu_priv *priv = policy->driver_data;108 struct cpufreq_frequency_table *p;109 unsigned int pstate;110 111 if (priv->info->cur_pstate_mask) {112 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS);113 114 pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift;115 } else {116 /*117 * For the fallback case we might not know the layout of DVFS_STATUS,118 * so just use the command register value (which ignores boost limitations).119 */120 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);121 122 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);123 }124 125 cpufreq_for_each_valid_entry(p, policy->freq_table)126 if (p->driver_data == pstate)127 return p->frequency;128 129 dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",130 pstate);131 return 0;132}133 134static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,135 unsigned int index)136{137 struct apple_cpu_priv *priv = policy->driver_data;138 unsigned int pstate = policy->freq_table[index].driver_data;139 u64 reg;140 141 /* Fallback for newer SoCs */142 if (index > priv->info->max_pstate)143 index = priv->info->max_pstate;144 145 if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,146 !(reg & APPLE_DVFS_CMD_BUSY), 2,147 APPLE_DVFS_TRANSITION_TIMEOUT)) {148 return -EIO;149 }150 151 reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2);152 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate);153 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);154 reg |= APPLE_DVFS_CMD_SET;155 156 writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);157 158 return 0;159}160 161static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,162 unsigned int target_freq)163{164 if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)165 return 0;166 167 return policy->freq_table[policy->cached_resolved_idx].frequency;168}169 170static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,171 void __iomem **reg_base,172 const struct apple_soc_cpufreq_info **info)173{174 struct of_phandle_args args;175 const struct of_device_id *match;176 int ret = 0;177 178 ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",179 "#performance-domain-cells",180 policy->cpus, &args);181 if (ret < 0)182 return ret;183 184 match = of_match_node(apple_soc_cpufreq_of_match, args.np);185 of_node_put(args.np);186 if (!match)187 return -ENODEV;188 189 *info = match->data;190 191 *reg_base = of_iomap(args.np, 0);192 if (!*reg_base)193 return -ENOMEM;194 195 return 0;196}197 198static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {199 &cpufreq_freq_attr_scaling_available_freqs,200 NULL, /* Filled in below if boost is enabled */201 NULL,202};203 204static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)205{206 int ret, i;207 unsigned int transition_latency;208 void __iomem *reg_base;209 struct device *cpu_dev;210 struct apple_cpu_priv *priv;211 const struct apple_soc_cpufreq_info *info;212 struct cpufreq_frequency_table *freq_table;213 214 cpu_dev = get_cpu_device(policy->cpu);215 if (!cpu_dev) {216 pr_err("failed to get cpu%d device\n", policy->cpu);217 return -ENODEV;218 }219 220 ret = dev_pm_opp_of_add_table(cpu_dev);221 if (ret < 0) {222 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);223 return ret;224 }225 226 ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info);227 if (ret) {228 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);229 return ret;230 }231 232 ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);233 if (ret) {234 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);235 goto out_iounmap;236 }237 238 ret = dev_pm_opp_get_opp_count(cpu_dev);239 if (ret <= 0) {240 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");241 ret = -EPROBE_DEFER;242 goto out_free_opp;243 }244 245 priv = kzalloc(sizeof(*priv), GFP_KERNEL);246 if (!priv) {247 ret = -ENOMEM;248 goto out_free_opp;249 }250 251 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);252 if (ret) {253 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);254 goto out_free_priv;255 }256 257 /* Get OPP levels (p-state indexes) and stash them in driver_data */258 for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {259 unsigned long rate = freq_table[i].frequency * 1000 + 999;260 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);261 262 if (IS_ERR(opp)) {263 ret = PTR_ERR(opp);264 goto out_free_cpufreq_table;265 }266 freq_table[i].driver_data = dev_pm_opp_get_level(opp);267 dev_pm_opp_put(opp);268 }269 270 priv->cpu_dev = cpu_dev;271 priv->reg_base = reg_base;272 priv->info = info;273 policy->driver_data = priv;274 policy->freq_table = freq_table;275 276 transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);277 if (!transition_latency)278 transition_latency = CPUFREQ_ETERNAL;279 280 policy->cpuinfo.transition_latency = transition_latency;281 policy->dvfs_possible_from_any_cpu = true;282 policy->fast_switch_possible = true;283 policy->suspend_freq = freq_table[0].frequency;284 285 if (policy_has_boost_freq(policy)) {286 ret = cpufreq_enable_boost_support();287 if (ret) {288 dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);289 } else {290 apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;291 apple_soc_cpufreq_driver.boost_enabled = true;292 }293 }294 295 return 0;296 297out_free_cpufreq_table:298 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);299out_free_priv:300 kfree(priv);301out_free_opp:302 dev_pm_opp_remove_all_dynamic(cpu_dev);303out_iounmap:304 iounmap(reg_base);305 return ret;306}307 308static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy)309{310 struct apple_cpu_priv *priv = policy->driver_data;311 312 dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);313 dev_pm_opp_remove_all_dynamic(priv->cpu_dev);314 iounmap(priv->reg_base);315 kfree(priv);316}317 318static struct cpufreq_driver apple_soc_cpufreq_driver = {319 .name = "apple-cpufreq",320 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |321 CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,322 .verify = cpufreq_generic_frequency_table_verify,323 .get = apple_soc_cpufreq_get_rate,324 .init = apple_soc_cpufreq_init,325 .exit = apple_soc_cpufreq_exit,326 .target_index = apple_soc_cpufreq_set_target,327 .fast_switch = apple_soc_cpufreq_fast_switch,328 .register_em = cpufreq_register_em_with_opp,329 .attr = apple_soc_cpufreq_hw_attr,330 .suspend = cpufreq_generic_suspend,331};332 333static int __init apple_soc_cpufreq_module_init(void)334{335 if (!of_machine_is_compatible("apple,arm-platform"))336 return -ENODEV;337 338 return cpufreq_register_driver(&apple_soc_cpufreq_driver);339}340module_init(apple_soc_cpufreq_module_init);341 342static void __exit apple_soc_cpufreq_module_exit(void)343{344 cpufreq_unregister_driver(&apple_soc_cpufreq_driver);345}346module_exit(apple_soc_cpufreq_module_exit);347 348MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);349MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");350MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");351MODULE_LICENSE("GPL");352