510 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2//3// Copyright (C) 2020-2021 Samuel Holland <samuel@sholland.org>4//5 6#include <linux/clk.h>7#include <linux/devfreq.h>8#include <linux/err.h>9#include <linux/io.h>10#include <linux/iopoll.h>11#include <linux/module.h>12#include <linux/of.h>13#include <linux/platform_device.h>14#include <linux/property.h>15 16#define MBUS_CR 0x000017#define MBUS_CR_GET_DRAM_TYPE(x) (((x) >> 16) & 0x7)18#define MBUS_CR_DRAM_TYPE_DDR2 219#define MBUS_CR_DRAM_TYPE_DDR3 320#define MBUS_CR_DRAM_TYPE_DDR4 421#define MBUS_CR_DRAM_TYPE_LPDDR2 622#define MBUS_CR_DRAM_TYPE_LPDDR3 723 24#define MBUS_TMR 0x000c25#define MBUS_TMR_PERIOD(x) ((x) - 1)26 27#define MBUS_PMU_CFG 0x009c28#define MBUS_PMU_CFG_PERIOD(x) (((x) - 1) << 16)29#define MBUS_PMU_CFG_UNIT (0x3 << 1)30#define MBUS_PMU_CFG_UNIT_B (0x0 << 1)31#define MBUS_PMU_CFG_UNIT_KB (0x1 << 1)32#define MBUS_PMU_CFG_UNIT_MB (0x2 << 1)33#define MBUS_PMU_CFG_ENABLE (0x1 << 0)34 35#define MBUS_PMU_BWCR(n) (0x00a0 + (0x04 * (n)))36 37#define MBUS_TOTAL_BWCR MBUS_PMU_BWCR(5)38#define MBUS_TOTAL_BWCR_H616 MBUS_PMU_BWCR(13)39 40#define MBUS_MDFSCR 0x010041#define MBUS_MDFSCR_BUFFER_TIMING (0x1 << 15)42#define MBUS_MDFSCR_PAD_HOLD (0x1 << 13)43#define MBUS_MDFSCR_BYPASS (0x1 << 4)44#define MBUS_MDFSCR_MODE (0x1 << 1)45#define MBUS_MDFSCR_MODE_DFS (0x0 << 1)46#define MBUS_MDFSCR_MODE_CFS (0x1 << 1)47#define MBUS_MDFSCR_START (0x1 << 0)48 49#define MBUS_MDFSMRMR 0x010850 51#define DRAM_PWRCTL 0x000452#define DRAM_PWRCTL_SELFREF_EN (0x1 << 0)53 54#define DRAM_RFSHTMG 0x009055#define DRAM_RFSHTMG_TREFI(x) ((x) << 16)56#define DRAM_RFSHTMG_TRFC(x) ((x) << 0)57 58#define DRAM_VTFCR 0x00b859#define DRAM_VTFCR_VTF_ENABLE (0x3 << 8)60 61#define DRAM_ODTMAP 0x012062 63#define DRAM_DX_MAX 464 65#define DRAM_DXnGCR0(n) (0x0344 + 0x80 * (n))66#define DRAM_DXnGCR0_DXODT (0x3 << 4)67#define DRAM_DXnGCR0_DXODT_DYNAMIC (0x0 << 4)68#define DRAM_DXnGCR0_DXODT_ENABLED (0x1 << 4)69#define DRAM_DXnGCR0_DXODT_DISABLED (0x2 << 4)70#define DRAM_DXnGCR0_DXEN (0x1 << 0)71 72struct sun8i_a33_mbus_variant {73 u32 min_dram_divider;74 u32 max_dram_divider;75 u32 odt_freq_mhz;76};77 78struct sun8i_a33_mbus {79 const struct sun8i_a33_mbus_variant *variant;80 void __iomem *reg_dram;81 void __iomem *reg_mbus;82 struct clk *clk_bus;83 struct clk *clk_dram;84 struct clk *clk_mbus;85 struct devfreq *devfreq_dram;86 struct devfreq_simple_ondemand_data gov_data;87 struct devfreq_dev_profile profile;88 u32 data_width;89 u32 nominal_bw;90 u32 odtmap;91 u32 tREFI_ns;92 u32 tRFC_ns;93 unsigned long freq_table[];94};95 96/*97 * The unit for this value is (MBUS clock cycles / MBUS_TMR_PERIOD). When98 * MBUS_TMR_PERIOD is programmed to match the MBUS clock frequency in MHz, as99 * it is during DRAM init and during probe, the resulting unit is microseconds.100 */101static int pmu_period = 50000;102module_param(pmu_period, int, 0644);103MODULE_PARM_DESC(pmu_period, "Bandwidth measurement period (microseconds)");104 105static u32 sun8i_a33_mbus_get_peak_bw(struct sun8i_a33_mbus *priv)106{107 /* Returns the peak transfer (in KiB) during any single PMU period. */108 return readl_relaxed(priv->reg_mbus + MBUS_TOTAL_BWCR);109}110 111static void sun8i_a33_mbus_restart_pmu_counters(struct sun8i_a33_mbus *priv)112{113 u32 pmu_cfg = MBUS_PMU_CFG_PERIOD(pmu_period) | MBUS_PMU_CFG_UNIT_KB;114 115 /* All PMU counters are cleared on a disable->enable transition. */116 writel_relaxed(pmu_cfg,117 priv->reg_mbus + MBUS_PMU_CFG);118 writel_relaxed(pmu_cfg | MBUS_PMU_CFG_ENABLE,119 priv->reg_mbus + MBUS_PMU_CFG);120 121}122 123static void sun8i_a33_mbus_update_nominal_bw(struct sun8i_a33_mbus *priv,124 u32 ddr_freq_mhz)125{126 /*127 * Nominal bandwidth (KiB per PMU period):128 *129 * DDR transfers microseconds KiB130 * ------------- * ------------ * --------131 * microsecond PMU period transfer132 */133 priv->nominal_bw = ddr_freq_mhz * pmu_period * priv->data_width / 1024;134}135 136static int sun8i_a33_mbus_set_dram_freq(struct sun8i_a33_mbus *priv,137 unsigned long freq)138{139 u32 ddr_freq_mhz = freq / USEC_PER_SEC; /* DDR */140 u32 dram_freq_mhz = ddr_freq_mhz / 2; /* SDR */141 u32 mctl_freq_mhz = dram_freq_mhz / 2; /* HDR */142 u32 dxodt, mdfscr, pwrctl, vtfcr;143 u32 i, tREFI_32ck, tRFC_ck;144 int ret;145 146 /* The rate change is not effective until the MDFS process runs. */147 ret = clk_set_rate(priv->clk_dram, freq);148 if (ret)149 return ret;150 151 /* Disable automatic self-refesh and VTF before starting MDFS. */152 pwrctl = readl_relaxed(priv->reg_dram + DRAM_PWRCTL) &153 ~DRAM_PWRCTL_SELFREF_EN;154 writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);155 vtfcr = readl_relaxed(priv->reg_dram + DRAM_VTFCR);156 writel_relaxed(vtfcr & ~DRAM_VTFCR_VTF_ENABLE,157 priv->reg_dram + DRAM_VTFCR);158 159 /* Set up MDFS and enable double buffering for timing registers. */160 mdfscr = MBUS_MDFSCR_MODE_DFS |161 MBUS_MDFSCR_BYPASS |162 MBUS_MDFSCR_PAD_HOLD |163 MBUS_MDFSCR_BUFFER_TIMING;164 writel(mdfscr, priv->reg_mbus + MBUS_MDFSCR);165 166 /* Update the buffered copy of RFSHTMG. */167 tREFI_32ck = priv->tREFI_ns * mctl_freq_mhz / 1000 / 32;168 tRFC_ck = DIV_ROUND_UP(priv->tRFC_ns * mctl_freq_mhz, 1000);169 writel(DRAM_RFSHTMG_TREFI(tREFI_32ck) | DRAM_RFSHTMG_TRFC(tRFC_ck),170 priv->reg_dram + DRAM_RFSHTMG);171 172 /* Enable ODT if needed, or disable it to save power. */173 if (priv->odtmap && dram_freq_mhz > priv->variant->odt_freq_mhz) {174 dxodt = DRAM_DXnGCR0_DXODT_DYNAMIC;175 writel(priv->odtmap, priv->reg_dram + DRAM_ODTMAP);176 } else {177 dxodt = DRAM_DXnGCR0_DXODT_DISABLED;178 writel(0, priv->reg_dram + DRAM_ODTMAP);179 }180 for (i = 0; i < DRAM_DX_MAX; ++i) {181 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);182 183 writel((readl(reg) & ~DRAM_DXnGCR0_DXODT) | dxodt, reg);184 }185 186 dev_dbg(priv->devfreq_dram->dev.parent,187 "Setting DRAM to %u MHz, tREFI=%u, tRFC=%u, ODT=%s\n",188 dram_freq_mhz, tREFI_32ck, tRFC_ck,189 dxodt == DRAM_DXnGCR0_DXODT_DYNAMIC ? "dynamic" : "disabled");190 191 /* Trigger hardware MDFS. */192 writel(mdfscr | MBUS_MDFSCR_START, priv->reg_mbus + MBUS_MDFSCR);193 ret = readl_poll_timeout_atomic(priv->reg_mbus + MBUS_MDFSCR, mdfscr,194 !(mdfscr & MBUS_MDFSCR_START), 10, 1000);195 if (ret)196 return ret;197 198 /* Disable double buffering. */199 writel(0, priv->reg_mbus + MBUS_MDFSCR);200 201 /* Restore VTF configuration. */202 writel_relaxed(vtfcr, priv->reg_dram + DRAM_VTFCR);203 204 /* Enable automatic self-refresh at the lowest frequency only. */205 if (freq == priv->freq_table[0])206 pwrctl |= DRAM_PWRCTL_SELFREF_EN;207 writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);208 209 sun8i_a33_mbus_restart_pmu_counters(priv);210 sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq_mhz);211 212 return 0;213}214 215static int sun8i_a33_mbus_set_dram_target(struct device *dev,216 unsigned long *freq, u32 flags)217{218 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);219 struct devfreq *devfreq = priv->devfreq_dram;220 struct dev_pm_opp *opp;221 int ret;222 223 opp = devfreq_recommended_opp(dev, freq, flags);224 if (IS_ERR(opp))225 return PTR_ERR(opp);226 227 dev_pm_opp_put(opp);228 229 if (*freq == devfreq->previous_freq)230 return 0;231 232 ret = sun8i_a33_mbus_set_dram_freq(priv, *freq);233 if (ret) {234 dev_warn(dev, "failed to set DRAM frequency: %d\n", ret);235 *freq = devfreq->previous_freq;236 }237 238 return ret;239}240 241static int sun8i_a33_mbus_get_dram_status(struct device *dev,242 struct devfreq_dev_status *stat)243{244 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);245 246 stat->busy_time = sun8i_a33_mbus_get_peak_bw(priv);247 stat->total_time = priv->nominal_bw;248 stat->current_frequency = priv->devfreq_dram->previous_freq;249 250 sun8i_a33_mbus_restart_pmu_counters(priv);251 252 dev_dbg(dev, "Using %lu/%lu (%lu%%) at %lu MHz\n",253 stat->busy_time, stat->total_time,254 DIV_ROUND_CLOSEST(stat->busy_time * 100, stat->total_time),255 stat->current_frequency / USEC_PER_SEC);256 257 return 0;258}259 260static int sun8i_a33_mbus_hw_init(struct device *dev,261 struct sun8i_a33_mbus *priv,262 unsigned long ddr_freq)263{264 u32 i, mbus_cr, mbus_freq_mhz;265 266 /* Choose tREFI and tRFC to match the configured DRAM type. */267 mbus_cr = readl_relaxed(priv->reg_mbus + MBUS_CR);268 switch (MBUS_CR_GET_DRAM_TYPE(mbus_cr)) {269 case MBUS_CR_DRAM_TYPE_DDR2:270 case MBUS_CR_DRAM_TYPE_DDR3:271 case MBUS_CR_DRAM_TYPE_DDR4:272 priv->tREFI_ns = 7800;273 priv->tRFC_ns = 350;274 break;275 case MBUS_CR_DRAM_TYPE_LPDDR2:276 case MBUS_CR_DRAM_TYPE_LPDDR3:277 priv->tREFI_ns = 3900;278 priv->tRFC_ns = 210;279 break;280 default:281 return -EINVAL;282 }283 284 /* Save ODTMAP so it can be restored when raising the frequency. */285 priv->odtmap = readl_relaxed(priv->reg_dram + DRAM_ODTMAP);286 287 /* Compute the DRAM data bus width by counting enabled DATx8 blocks. */288 for (i = 0; i < DRAM_DX_MAX; ++i) {289 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);290 291 if (!(readl_relaxed(reg) & DRAM_DXnGCR0_DXEN))292 break;293 }294 priv->data_width = i;295 296 dev_dbg(dev, "Detected %u-bit %sDDRx with%s ODT\n",297 priv->data_width * 8,298 MBUS_CR_GET_DRAM_TYPE(mbus_cr) > 4 ? "LP" : "",299 priv->odtmap ? "" : "out");300 301 /* Program MBUS_TMR such that the PMU period unit is microseconds. */302 mbus_freq_mhz = clk_get_rate(priv->clk_mbus) / USEC_PER_SEC;303 writel_relaxed(MBUS_TMR_PERIOD(mbus_freq_mhz),304 priv->reg_mbus + MBUS_TMR);305 306 /* "Master Ready Mask Register" bits must be set or MDFS will block. */307 writel_relaxed(0xffffffff, priv->reg_mbus + MBUS_MDFSMRMR);308 309 sun8i_a33_mbus_restart_pmu_counters(priv);310 sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq / USEC_PER_SEC);311 312 return 0;313}314 315static int __maybe_unused sun8i_a33_mbus_suspend(struct device *dev)316{317 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);318 319 clk_disable_unprepare(priv->clk_bus);320 321 return 0;322}323 324static int __maybe_unused sun8i_a33_mbus_resume(struct device *dev)325{326 struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);327 328 return clk_prepare_enable(priv->clk_bus);329}330 331static int sun8i_a33_mbus_probe(struct platform_device *pdev)332{333 const struct sun8i_a33_mbus_variant *variant;334 struct device *dev = &pdev->dev;335 struct sun8i_a33_mbus *priv;336 unsigned long base_freq;337 unsigned int max_state;338 const char *err;339 int i, ret;340 341 variant = device_get_match_data(dev);342 if (!variant)343 return -EINVAL;344 345 max_state = variant->max_dram_divider - variant->min_dram_divider + 1;346 347 priv = devm_kzalloc(dev, struct_size(priv, freq_table, max_state), GFP_KERNEL);348 if (!priv)349 return -ENOMEM;350 351 platform_set_drvdata(pdev, priv);352 353 priv->variant = variant;354 355 priv->reg_dram = devm_platform_ioremap_resource_byname(pdev, "dram");356 if (IS_ERR(priv->reg_dram))357 return PTR_ERR(priv->reg_dram);358 359 priv->reg_mbus = devm_platform_ioremap_resource_byname(pdev, "mbus");360 if (IS_ERR(priv->reg_mbus))361 return PTR_ERR(priv->reg_mbus);362 363 priv->clk_bus = devm_clk_get(dev, "bus");364 if (IS_ERR(priv->clk_bus))365 return dev_err_probe(dev, PTR_ERR(priv->clk_bus),366 "failed to get bus clock\n");367 368 priv->clk_dram = devm_clk_get(dev, "dram");369 if (IS_ERR(priv->clk_dram))370 return dev_err_probe(dev, PTR_ERR(priv->clk_dram),371 "failed to get dram clock\n");372 373 priv->clk_mbus = devm_clk_get(dev, "mbus");374 if (IS_ERR(priv->clk_mbus))375 return dev_err_probe(dev, PTR_ERR(priv->clk_mbus),376 "failed to get mbus clock\n");377 378 ret = clk_prepare_enable(priv->clk_bus);379 if (ret)380 return dev_err_probe(dev, ret,381 "failed to enable bus clock\n");382 383 /* Lock the DRAM clock rate to keep priv->nominal_bw in sync. */384 ret = clk_rate_exclusive_get(priv->clk_dram);385 if (ret) {386 err = "failed to lock dram clock rate\n";387 goto err_disable_bus;388 }389 390 /* Lock the MBUS clock rate to keep MBUS_TMR_PERIOD in sync. */391 ret = clk_rate_exclusive_get(priv->clk_mbus);392 if (ret) {393 err = "failed to lock mbus clock rate\n";394 goto err_unlock_dram;395 }396 397 priv->gov_data.upthreshold = 10;398 priv->gov_data.downdifferential = 5;399 400 priv->profile.initial_freq = clk_get_rate(priv->clk_dram);401 priv->profile.polling_ms = 1000;402 priv->profile.target = sun8i_a33_mbus_set_dram_target;403 priv->profile.get_dev_status = sun8i_a33_mbus_get_dram_status;404 priv->profile.freq_table = priv->freq_table;405 priv->profile.max_state = max_state;406 407 ret = devm_pm_opp_set_clkname(dev, "dram");408 if (ret) {409 err = "failed to add OPP table\n";410 goto err_unlock_mbus;411 }412 413 base_freq = clk_get_rate(clk_get_parent(priv->clk_dram));414 for (i = 0; i < max_state; ++i) {415 unsigned int div = variant->max_dram_divider - i;416 417 priv->freq_table[i] = base_freq / div;418 419 ret = dev_pm_opp_add(dev, priv->freq_table[i], 0);420 if (ret) {421 err = "failed to add OPPs\n";422 goto err_remove_opps;423 }424 }425 426 ret = sun8i_a33_mbus_hw_init(dev, priv, priv->profile.initial_freq);427 if (ret) {428 err = "failed to init hardware\n";429 goto err_remove_opps;430 }431 432 priv->devfreq_dram = devfreq_add_device(dev, &priv->profile,433 DEVFREQ_GOV_SIMPLE_ONDEMAND,434 &priv->gov_data);435 if (IS_ERR(priv->devfreq_dram)) {436 ret = PTR_ERR(priv->devfreq_dram);437 err = "failed to add devfreq device\n";438 goto err_remove_opps;439 }440 441 /*442 * This must be set manually after registering the devfreq device,443 * because there is no way to select a dynamic OPP as the suspend OPP.444 */445 priv->devfreq_dram->suspend_freq = priv->freq_table[0];446 447 return 0;448 449err_remove_opps:450 dev_pm_opp_remove_all_dynamic(dev);451err_unlock_mbus:452 clk_rate_exclusive_put(priv->clk_mbus);453err_unlock_dram:454 clk_rate_exclusive_put(priv->clk_dram);455err_disable_bus:456 clk_disable_unprepare(priv->clk_bus);457 458 return dev_err_probe(dev, ret, err);459}460 461static void sun8i_a33_mbus_remove(struct platform_device *pdev)462{463 struct sun8i_a33_mbus *priv = platform_get_drvdata(pdev);464 unsigned long initial_freq = priv->profile.initial_freq;465 struct device *dev = &pdev->dev;466 int ret;467 468 devfreq_remove_device(priv->devfreq_dram);469 470 ret = sun8i_a33_mbus_set_dram_freq(priv, initial_freq);471 if (ret)472 dev_warn(dev, "failed to restore DRAM frequency: %d\n", ret);473 474 dev_pm_opp_remove_all_dynamic(dev);475 clk_rate_exclusive_put(priv->clk_mbus);476 clk_rate_exclusive_put(priv->clk_dram);477 clk_disable_unprepare(priv->clk_bus);478}479 480static const struct sun8i_a33_mbus_variant sun50i_a64_mbus = {481 .min_dram_divider = 1,482 .max_dram_divider = 4,483 .odt_freq_mhz = 400,484};485 486static const struct of_device_id sun8i_a33_mbus_of_match[] = {487 { .compatible = "allwinner,sun50i-a64-mbus", .data = &sun50i_a64_mbus },488 { .compatible = "allwinner,sun50i-h5-mbus", .data = &sun50i_a64_mbus },489 { },490};491MODULE_DEVICE_TABLE(of, sun8i_a33_mbus_of_match);492 493static SIMPLE_DEV_PM_OPS(sun8i_a33_mbus_pm_ops,494 sun8i_a33_mbus_suspend, sun8i_a33_mbus_resume);495 496static struct platform_driver sun8i_a33_mbus_driver = {497 .probe = sun8i_a33_mbus_probe,498 .remove_new = sun8i_a33_mbus_remove,499 .driver = {500 .name = "sun8i-a33-mbus",501 .of_match_table = sun8i_a33_mbus_of_match,502 .pm = pm_ptr(&sun8i_a33_mbus_pm_ops),503 },504};505module_platform_driver(sun8i_a33_mbus_driver);506 507MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");508MODULE_DESCRIPTION("Allwinner sun8i/sun50i MBUS DEVFREQ Driver");509MODULE_LICENSE("GPL v2");510