1584 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (c) 2019 Samsung Electronics Co., Ltd.4 * Author: Lukasz Luba <l.luba@partner.samsung.com>5 */6 7#include <linux/cleanup.h>8#include <linux/clk.h>9#include <linux/devfreq.h>10#include <linux/devfreq-event.h>11#include <linux/device.h>12#include <linux/interrupt.h>13#include <linux/io.h>14#include <linux/mfd/syscon.h>15#include <linux/module.h>16#include <linux/moduleparam.h>17#include <linux/of.h>18#include <linux/pm_opp.h>19#include <linux/platform_device.h>20#include <linux/regmap.h>21#include <linux/regulator/consumer.h>22#include <linux/slab.h>23#include "../jedec_ddr.h"24#include "../of_memory.h"25 26static int irqmode;27module_param(irqmode, int, 0644);28MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)");29 30#define EXYNOS5_DREXI_TIMINGAREF (0x0030)31#define EXYNOS5_DREXI_TIMINGROW0 (0x0034)32#define EXYNOS5_DREXI_TIMINGDATA0 (0x0038)33#define EXYNOS5_DREXI_TIMINGPOWER0 (0x003C)34#define EXYNOS5_DREXI_TIMINGROW1 (0x00E4)35#define EXYNOS5_DREXI_TIMINGDATA1 (0x00E8)36#define EXYNOS5_DREXI_TIMINGPOWER1 (0x00EC)37#define CDREX_PAUSE (0x2091c)38#define CDREX_LPDDR3PHY_CON3 (0x20a20)39#define CDREX_LPDDR3PHY_CLKM_SRC (0x20700)40#define EXYNOS5_TIMING_SET_SWI BIT(28)41#define USE_MX_MSPLL_TIMINGS (1)42#define USE_BPLL_TIMINGS (0)43#define EXYNOS5_AREF_NORMAL (0x2e)44 45#define DREX_PPCCLKCON (0x0130)46#define DREX_PEREV2CONFIG (0x013c)47#define DREX_PMNC_PPC (0xE000)48#define DREX_CNTENS_PPC (0xE010)49#define DREX_CNTENC_PPC (0xE020)50#define DREX_INTENS_PPC (0xE030)51#define DREX_INTENC_PPC (0xE040)52#define DREX_FLAG_PPC (0xE050)53#define DREX_PMCNT2_PPC (0xE130)54 55/*56 * A value for register DREX_PMNC_PPC which should be written to reset57 * the cycle counter CCNT (a reference wall clock). It sets zero to the58 * CCNT counter.59 */60#define CC_RESET BIT(2)61 62/*63 * A value for register DREX_PMNC_PPC which does the reset of all performance64 * counters to zero.65 */66#define PPC_COUNTER_RESET BIT(1)67 68/*69 * Enables all configured counters (including cycle counter). The value should70 * be written to the register DREX_PMNC_PPC.71 */72#define PPC_ENABLE BIT(0)73 74/* A value for register DREX_PPCCLKCON which enables performance events clock.75 * Must be written before first access to the performance counters register76 * set, otherwise it could crash.77 */78#define PEREV_CLK_EN BIT(0)79 80/*81 * Values which are used to enable counters, interrupts or configure flags of82 * the performance counters. They configure counter 2 and cycle counter.83 */84#define PERF_CNT2 BIT(2)85#define PERF_CCNT BIT(31)86 87/*88 * Performance event types which are used for setting the preferred event89 * to track in the counters.90 * There is a set of different types, the values are from range 0 to 0x6f.91 * These settings should be written to the configuration register which manages92 * the type of the event (register DREX_PEREV2CONFIG).93 */94#define READ_TRANSFER_CH0 (0x6d)95#define READ_TRANSFER_CH1 (0x6f)96 97#define PERF_COUNTER_START_VALUE 0xff00000098#define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL99 100/**101 * struct dmc_opp_table - Operating level desciption102 * @freq_hz: target frequency in Hz103 * @volt_uv: target voltage in uV104 *105 * Covers frequency and voltage settings of the DMC operating mode.106 */107struct dmc_opp_table {108 u32 freq_hz;109 u32 volt_uv;110};111 112/**113 * struct exynos5_dmc - main structure describing DMC device114 * @dev: DMC device115 * @df: devfreq device structure returned by devfreq framework116 * @gov_data: configuration of devfreq governor117 * @base_drexi0: DREX0 registers mapping118 * @base_drexi1: DREX1 registers mapping119 * @clk_regmap: regmap for clock controller registers120 * @lock: protects curr_rate and frequency/voltage setting section121 * @curr_rate: current frequency122 * @curr_volt: current voltage123 * @opp: OPP table124 * @opp_count: number of 'opp' elements125 * @timings_arr_size: number of 'timings' elements126 * @timing_row: values for timing row register, for each OPP127 * @timing_data: values for timing data register, for each OPP128 * @timing_power: balues for timing power register, for each OPP129 * @timings: DDR memory timings, from device tree130 * @min_tck: DDR memory minimum timing values, from device tree131 * @bypass_timing_row: value for timing row register for bypass timings132 * @bypass_timing_data: value for timing data register for bypass timings133 * @bypass_timing_power: value for timing power register for bypass134 * timings135 * @vdd_mif: Memory interface regulator136 * @fout_spll: clock: SPLL137 * @fout_bpll: clock: BPLL138 * @mout_spll: clock: mux SPLL139 * @mout_bpll: clock: mux BPLL140 * @mout_mclk_cdrex: clock: mux mclk_cdrex141 * @mout_mx_mspll_ccore: clock: mux mx_mspll_ccore142 * @counter: devfreq events143 * @num_counters: number of 'counter' elements144 * @last_overflow_ts: time (in ns) of last overflow of each DREX145 * @load: utilization in percents146 * @total: total time between devfreq events147 * @in_irq_mode: whether running in interrupt mode (true)148 * or polling (false)149 *150 * The main structure for the Dynamic Memory Controller which covers clocks,151 * memory regions, HW information, parameters and current operating mode.152 */153struct exynos5_dmc {154 struct device *dev;155 struct devfreq *df;156 struct devfreq_simple_ondemand_data gov_data;157 void __iomem *base_drexi0;158 void __iomem *base_drexi1;159 struct regmap *clk_regmap;160 /* Protects curr_rate and frequency/voltage setting section */161 struct mutex lock;162 unsigned long curr_rate;163 unsigned long curr_volt;164 struct dmc_opp_table *opp;165 int opp_count;166 u32 timings_arr_size;167 u32 *timing_row;168 u32 *timing_data;169 u32 *timing_power;170 const struct lpddr3_timings *timings;171 const struct lpddr3_min_tck *min_tck;172 u32 bypass_timing_row;173 u32 bypass_timing_data;174 u32 bypass_timing_power;175 struct regulator *vdd_mif;176 struct clk *fout_spll;177 struct clk *fout_bpll;178 struct clk *mout_spll;179 struct clk *mout_bpll;180 struct clk *mout_mclk_cdrex;181 struct clk *mout_mx_mspll_ccore;182 struct devfreq_event_dev **counter;183 int num_counters;184 u64 last_overflow_ts[2];185 unsigned long load;186 unsigned long total;187 bool in_irq_mode;188};189 190#define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \191 { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }192 193#define TIMING_VAL2REG(timing, t_val) \194({ \195 u32 __val; \196 __val = (t_val) << (timing)->bit_beg; \197 __val; \198})199 200struct timing_reg {201 char *name;202 int bit_beg;203 int bit_end;204 unsigned int val;205};206 207static const struct timing_reg timing_row_reg_fields[] = {208 TIMING_FIELD("tRFC", 24, 31),209 TIMING_FIELD("tRRD", 20, 23),210 TIMING_FIELD("tRP", 16, 19),211 TIMING_FIELD("tRCD", 12, 15),212 TIMING_FIELD("tRC", 6, 11),213 TIMING_FIELD("tRAS", 0, 5),214};215 216static const struct timing_reg timing_data_reg_fields[] = {217 TIMING_FIELD("tWTR", 28, 31),218 TIMING_FIELD("tWR", 24, 27),219 TIMING_FIELD("tRTP", 20, 23),220 TIMING_FIELD("tW2W-C2C", 14, 14),221 TIMING_FIELD("tR2R-C2C", 12, 12),222 TIMING_FIELD("WL", 8, 11),223 TIMING_FIELD("tDQSCK", 4, 7),224 TIMING_FIELD("RL", 0, 3),225};226 227static const struct timing_reg timing_power_reg_fields[] = {228 TIMING_FIELD("tFAW", 26, 31),229 TIMING_FIELD("tXSR", 16, 25),230 TIMING_FIELD("tXP", 8, 15),231 TIMING_FIELD("tCKE", 4, 7),232 TIMING_FIELD("tMRD", 0, 3),233};234 235#define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \236 ARRAY_SIZE(timing_data_reg_fields) + \237 ARRAY_SIZE(timing_power_reg_fields))238 239static int exynos5_counters_set_event(struct exynos5_dmc *dmc)240{241 int i, ret;242 243 for (i = 0; i < dmc->num_counters; i++) {244 if (!dmc->counter[i])245 continue;246 ret = devfreq_event_set_event(dmc->counter[i]);247 if (ret < 0)248 return ret;249 }250 return 0;251}252 253static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)254{255 int i, ret;256 257 for (i = 0; i < dmc->num_counters; i++) {258 if (!dmc->counter[i])259 continue;260 ret = devfreq_event_enable_edev(dmc->counter[i]);261 if (ret < 0)262 return ret;263 }264 return 0;265}266 267static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)268{269 int i, ret;270 271 for (i = 0; i < dmc->num_counters; i++) {272 if (!dmc->counter[i])273 continue;274 ret = devfreq_event_disable_edev(dmc->counter[i]);275 if (ret < 0)276 return ret;277 }278 return 0;279}280 281/**282 * find_target_freq_idx() - Finds requested frequency in local DMC configuration283 * @dmc: device for which the information is checked284 * @target_rate: requested frequency in KHz285 *286 * Seeks in the local DMC driver structure for the requested frequency value287 * and returns index or error value.288 */289static int find_target_freq_idx(struct exynos5_dmc *dmc,290 unsigned long target_rate)291{292 int i;293 294 for (i = dmc->opp_count - 1; i >= 0; i--)295 if (dmc->opp[i].freq_hz <= target_rate)296 return i;297 298 return -EINVAL;299}300 301/**302 * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings303 * @dmc: device for which the new settings is going to be applied304 * @set: boolean variable passing set value305 *306 * Changes the register set, which holds timing parameters.307 * There is two register sets: 0 and 1. The register set 0308 * is used in normal operation when the clock is provided from main PLL.309 * The bank register set 1 is used when the main PLL frequency is going to be310 * changed and the clock is taken from alternative, stable source.311 * This function switches between these banks according to the312 * currently used clock source.313 */314static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)315{316 unsigned int reg;317 int ret;318 319 ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, ®);320 if (ret)321 return ret;322 323 if (set)324 reg |= EXYNOS5_TIMING_SET_SWI;325 else326 reg &= ~EXYNOS5_TIMING_SET_SWI;327 328 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);329 330 return 0;331}332 333/**334 * exynos5_init_freq_table() - Initialized PM OPP framework335 * @dmc: DMC device for which the frequencies are used for OPP init336 * @profile: devfreq device's profile337 *338 * Populate the devfreq device's OPP table based on current frequency, voltage.339 */340static int exynos5_init_freq_table(struct exynos5_dmc *dmc,341 struct devfreq_dev_profile *profile)342{343 struct device *dev = dmc->dev;344 int i, ret;345 int idx;346 unsigned long freq;347 348 ret = devm_pm_opp_of_add_table(dev);349 if (ret < 0) {350 dev_err(dev, "Failed to get OPP table\n");351 return ret;352 }353 354 dmc->opp_count = dev_pm_opp_get_opp_count(dev);355 356 dmc->opp = devm_kmalloc_array(dev, dmc->opp_count,357 sizeof(struct dmc_opp_table), GFP_KERNEL);358 if (!dmc->opp)359 return -ENOMEM;360 361 idx = dmc->opp_count - 1;362 for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {363 struct dev_pm_opp *opp;364 365 opp = dev_pm_opp_find_freq_floor(dev, &freq);366 if (IS_ERR(opp))367 return PTR_ERR(opp);368 369 dmc->opp[idx - i].freq_hz = freq;370 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);371 372 dev_pm_opp_put(opp);373 }374 375 return 0;376}377 378/**379 * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings380 * @dmc: device for which the new settings is going to be applied381 *382 * Low-level function for changing timings for DRAM memory clocking from383 * 'bypass' clock source (fixed frequency @400MHz).384 * It uses timing bank registers set 1.385 */386static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)387{388 writel(EXYNOS5_AREF_NORMAL,389 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);390 391 writel(dmc->bypass_timing_row,392 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);393 writel(dmc->bypass_timing_row,394 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);395 writel(dmc->bypass_timing_data,396 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);397 writel(dmc->bypass_timing_data,398 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);399 writel(dmc->bypass_timing_power,400 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);401 writel(dmc->bypass_timing_power,402 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);403}404 405/**406 * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings407 * @dmc: device for which the new settings is going to be applied408 * @target_rate: target frequency of the DMC409 *410 * Low-level function for changing timings for DRAM memory operating from main411 * clock source (BPLL), which can have different frequencies. Thus, each412 * frequency must have corresponding timings register values in order to keep413 * the needed delays.414 * It uses timing bank registers set 0.415 */416static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,417 unsigned long target_rate)418{419 int idx;420 421 for (idx = dmc->opp_count - 1; idx >= 0; idx--)422 if (dmc->opp[idx].freq_hz <= target_rate)423 break;424 425 if (idx < 0)426 return -EINVAL;427 428 writel(EXYNOS5_AREF_NORMAL,429 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);430 431 writel(dmc->timing_row[idx],432 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);433 writel(dmc->timing_row[idx],434 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);435 writel(dmc->timing_data[idx],436 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);437 writel(dmc->timing_data[idx],438 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);439 writel(dmc->timing_power[idx],440 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);441 writel(dmc->timing_power[idx],442 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);443 444 return 0;445}446 447/**448 * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC449 * @dmc: device for which it is going to be set450 * @target_volt: new voltage which is chosen to be final451 *452 * Function tries to align voltage to the safe level for 'normal' mode.453 * It checks the need of higher voltage and changes the value. The target454 * voltage might be lower that currently set and still the system will be455 * stable.456 */457static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,458 unsigned long target_volt)459{460 int ret = 0;461 462 if (dmc->curr_volt <= target_volt)463 return 0;464 465 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,466 target_volt);467 if (!ret)468 dmc->curr_volt = target_volt;469 470 return ret;471}472 473/**474 * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC475 * @dmc: device for which it is going to be set476 * @target_volt: new voltage which is chosen to be final477 *478 * Function tries to align voltage to the safe level for the 'bypass' mode.479 * It checks the need of higher voltage and changes the value.480 * The target voltage must not be less than currently needed, because481 * for current frequency the device might become unstable.482 */483static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,484 unsigned long target_volt)485{486 int ret = 0;487 488 if (dmc->curr_volt >= target_volt)489 return 0;490 491 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,492 target_volt);493 if (!ret)494 dmc->curr_volt = target_volt;495 496 return ret;497}498 499/**500 * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings501 * @dmc: device for which it is going to be set502 * @target_rate: new frequency which is chosen to be final503 *504 * Function changes the DRAM timings for the temporary 'bypass' mode.505 */506static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,507 unsigned long target_rate)508{509 int idx = find_target_freq_idx(dmc, target_rate);510 511 if (idx < 0)512 return -EINVAL;513 514 exynos5_set_bypass_dram_timings(dmc);515 516 return 0;517}518 519/**520 * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock521 * @dmc: DMC device for which the switching is going to happen522 * @target_rate: new frequency which is going to be set as a final523 * @target_volt: new voltage which is going to be set as a final524 *525 * Function configures DMC and clocks for operating in temporary 'bypass' mode.526 * This mode is used only temporary but if required, changes voltage and timings527 * for DRAM chips. It switches the main clock to stable clock source for the528 * period of the main PLL reconfiguration.529 */530static int531exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,532 unsigned long target_rate,533 unsigned long target_volt)534{535 int ret;536 537 /*538 * Having higher voltage for a particular frequency does not harm539 * the chip. Use it for the temporary frequency change when one540 * voltage manipulation might be avoided.541 */542 ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);543 if (ret)544 return ret;545 546 /*547 * Longer delays for DRAM does not cause crash, the opposite does.548 */549 ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);550 if (ret)551 return ret;552 553 /*554 * Delays are long enough, so use them for the new coming clock.555 */556 ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);557 558 return ret;559}560 561/**562 * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC563 * using safe procedure564 * @dmc: device for which the frequency is going to be changed565 * @target_rate: requested new frequency566 * @target_volt: requested voltage which corresponds to the new frequency567 *568 * The DMC frequency change procedure requires a few steps.569 * The main requirement is to change the clock source in the clk mux570 * for the time of main clock PLL locking. The assumption is that the571 * alternative clock source set as parent is stable.572 * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass'573 * clock. This requires alignment in DRAM timing parameters for the new574 * T-period. There is two bank sets for keeping DRAM575 * timings: set 0 and set 1. The set 0 is used when main clock source is576 * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between577 * the two bank sets is part of the process.578 * The voltage must also be aligned to the minimum required level. There is579 * this intermediate step with switching to 'bypass' parent clock source.580 * if the old voltage is lower, it requires an increase of the voltage level.581 * The complexity of the voltage manipulation is hidden in low level function.582 * In this function there is last alignment of the voltage level at the end.583 */584static int585exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,586 unsigned long target_rate,587 unsigned long target_volt)588{589 int ret;590 591 ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,592 target_volt);593 if (ret)594 return ret;595 596 /*597 * Voltage is set at least to a level needed for this frequency,598 * so switching clock source is safe now.599 */600 clk_prepare_enable(dmc->fout_spll);601 clk_prepare_enable(dmc->mout_spll);602 clk_prepare_enable(dmc->mout_mx_mspll_ccore);603 604 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);605 if (ret)606 goto disable_clocks;607 608 /*609 * We are safe to increase the timings for current bypass frequency.610 * Thanks to this the settings will be ready for the upcoming clock611 * source change.612 */613 exynos5_dram_change_timings(dmc, target_rate);614 615 clk_set_rate(dmc->fout_bpll, target_rate);616 617 ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);618 if (ret)619 goto disable_clocks;620 621 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);622 if (ret)623 goto disable_clocks;624 625 /*626 * Make sure if the voltage is not from 'bypass' settings and align to627 * the right level for power efficiency.628 */629 ret = exynos5_dmc_align_target_voltage(dmc, target_volt);630 631disable_clocks:632 clk_disable_unprepare(dmc->mout_mx_mspll_ccore);633 clk_disable_unprepare(dmc->mout_spll);634 clk_disable_unprepare(dmc->fout_spll);635 636 return ret;637}638 639/**640 * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP641 * table.642 * @dmc: device for which the frequency is going to be changed643 * @freq: requested frequency in KHz644 * @target_rate: returned frequency which is the same or lower than645 * requested646 * @target_volt: returned voltage which corresponds to the returned647 * frequency648 * @flags: devfreq flags provided for this frequency change request649 *650 * Function gets requested frequency and checks OPP framework for needed651 * frequency and voltage. It populates the values 'target_rate' and652 * 'target_volt' or returns error value when OPP framework fails.653 */654static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,655 unsigned long *freq,656 unsigned long *target_rate,657 unsigned long *target_volt, u32 flags)658{659 struct dev_pm_opp *opp;660 661 opp = devfreq_recommended_opp(dmc->dev, freq, flags);662 if (IS_ERR(opp))663 return PTR_ERR(opp);664 665 *target_rate = dev_pm_opp_get_freq(opp);666 *target_volt = dev_pm_opp_get_voltage(opp);667 dev_pm_opp_put(opp);668 669 return 0;670}671 672/**673 * exynos5_dmc_target() - Function responsible for changing frequency of DMC674 * @dev: device for which the frequency is going to be changed675 * @freq: requested frequency in KHz676 * @flags: flags provided for this frequency change request677 *678 * An entry function provided to the devfreq framework which provides frequency679 * change of the DMC. The function gets the possible rate from OPP table based680 * on requested frequency. It calls the next function responsible for the681 * frequency and voltage change. In case of failure, does not set 'curr_rate'682 * and returns error value to the framework.683 */684static int exynos5_dmc_target(struct device *dev, unsigned long *freq,685 u32 flags)686{687 struct exynos5_dmc *dmc = dev_get_drvdata(dev);688 unsigned long target_rate = 0;689 unsigned long target_volt = 0;690 int ret;691 692 ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,693 flags);694 695 if (ret)696 return ret;697 698 if (target_rate == dmc->curr_rate)699 return 0;700 701 mutex_lock(&dmc->lock);702 703 ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);704 705 if (ret) {706 mutex_unlock(&dmc->lock);707 return ret;708 }709 710 dmc->curr_rate = target_rate;711 712 mutex_unlock(&dmc->lock);713 return 0;714}715 716/**717 * exynos5_counters_get() - Gets the performance counters values.718 * @dmc: device for which the counters are going to be checked719 * @load_count: variable which is populated with counter value720 * @total_count: variable which is used as 'wall clock' reference721 *722 * Function which provides performance counters values. It sums up counters for723 * two DMC channels. The 'total_count' is used as a reference and max value.724 * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%].725 */726static int exynos5_counters_get(struct exynos5_dmc *dmc,727 unsigned long *load_count,728 unsigned long *total_count)729{730 unsigned long total = 0;731 struct devfreq_event_data event;732 int ret, i;733 734 *load_count = 0;735 736 /* Take into account only read+write counters, but stop all */737 for (i = 0; i < dmc->num_counters; i++) {738 if (!dmc->counter[i])739 continue;740 741 ret = devfreq_event_get_event(dmc->counter[i], &event);742 if (ret < 0)743 return ret;744 745 *load_count += event.load_count;746 747 if (total < event.total_count)748 total = event.total_count;749 }750 751 *total_count = total;752 753 return 0;754}755 756/**757 * exynos5_dmc_start_perf_events() - Setup and start performance event counters758 * @dmc: device for which the counters are going to be checked759 * @beg_value: initial value for the counter760 *761 * Function which enables needed counters, interrupts and sets initial values762 * then starts the counters.763 */764static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,765 u32 beg_value)766{767 /* Enable interrupts for counter 2 */768 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);769 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);770 771 /* Enable counter 2 and CCNT */772 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);773 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);774 775 /* Clear overflow flag for all counters */776 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);777 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);778 779 /* Reset all counters */780 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);781 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);782 783 /*784 * Set start value for the counters, the number of samples that785 * will be gathered is calculated as: 0xffffffff - beg_value786 */787 writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);788 writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);789 790 /* Start all counters */791 writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);792 writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);793}794 795/**796 * exynos5_dmc_perf_events_calc() - Calculate utilization797 * @dmc: device for which the counters are going to be checked798 * @diff_ts: time between last interrupt and current one799 *800 * Function which calculates needed utilization for the devfreq governor.801 * It prepares values for 'busy_time' and 'total_time' based on elapsed time802 * between interrupts, which approximates utilization.803 */804static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)805{806 /*807 * This is a simple algorithm for managing traffic on DMC.808 * When there is almost no load the counters overflow every 4s,809 * no mater the DMC frequency.810 * The high load might be approximated using linear function.811 * Knowing that, simple calculation can provide 'busy_time' and812 * 'total_time' to the devfreq governor which picks up target813 * frequency.814 * We want a fast ramp up and slow decay in frequency change function.815 */816 if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {817 /*818 * Set higher utilization for the simple_ondemand governor.819 * The governor should increase the frequency of the DMC.820 */821 dmc->load = 70;822 dmc->total = 100;823 } else {824 /*825 * Set low utilization for the simple_ondemand governor.826 * The governor should decrease the frequency of the DMC.827 */828 dmc->load = 35;829 dmc->total = 100;830 }831 832 dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);833}834 835/**836 * exynos5_dmc_perf_events_check() - Checks the status of the counters837 * @dmc: device for which the counters are going to be checked838 *839 * Function which is called from threaded IRQ to check the counters state840 * and to call approximation for the needed utilization.841 */842static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)843{844 u32 val;845 u64 diff_ts, ts;846 847 ts = ktime_get_ns();848 849 /* Stop all counters */850 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);851 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);852 853 /* Check the source in interrupt flag registers (which channel) */854 val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);855 if (val) {856 diff_ts = ts - dmc->last_overflow_ts[0];857 dmc->last_overflow_ts[0] = ts;858 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n", val);859 } else {860 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);861 diff_ts = ts - dmc->last_overflow_ts[1];862 dmc->last_overflow_ts[1] = ts;863 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n", val);864 }865 866 exynos5_dmc_perf_events_calc(dmc, diff_ts);867 868 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);869}870 871/**872 * exynos5_dmc_enable_perf_events() - Enable performance events873 * @dmc: device for which the counters are going to be checked874 *875 * Function which is setup needed environment and enables counters.876 */877static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)878{879 u64 ts;880 881 /* Enable Performance Event Clock */882 writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);883 writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);884 885 /* Select read transfers as performance event2 */886 writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);887 writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);888 889 ts = ktime_get_ns();890 dmc->last_overflow_ts[0] = ts;891 dmc->last_overflow_ts[1] = ts;892 893 /* Devfreq shouldn't be faster than initialization, play safe though. */894 dmc->load = 99;895 dmc->total = 100;896}897 898/**899 * exynos5_dmc_disable_perf_events() - Disable performance events900 * @dmc: device for which the counters are going to be checked901 *902 * Function which stops, disables performance event counters and interrupts.903 */904static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)905{906 /* Stop all counters */907 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);908 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);909 910 /* Disable interrupts for counter 2 */911 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);912 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);913 914 /* Disable counter 2 and CCNT */915 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);916 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);917 918 /* Clear overflow flag for all counters */919 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);920 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);921}922 923/**924 * exynos5_dmc_get_status() - Read current DMC performance statistics.925 * @dev: device for which the statistics are requested926 * @stat: structure which has statistic fields927 *928 * Function reads the DMC performance counters and calculates 'busy_time'929 * and 'total_time'. To protect from overflow, the values are shifted right930 * by 10. After read out the counters are setup to count again.931 */932static int exynos5_dmc_get_status(struct device *dev,933 struct devfreq_dev_status *stat)934{935 struct exynos5_dmc *dmc = dev_get_drvdata(dev);936 unsigned long load, total;937 int ret;938 939 if (dmc->in_irq_mode) {940 mutex_lock(&dmc->lock);941 stat->current_frequency = dmc->curr_rate;942 mutex_unlock(&dmc->lock);943 944 stat->busy_time = dmc->load;945 stat->total_time = dmc->total;946 } else {947 ret = exynos5_counters_get(dmc, &load, &total);948 if (ret < 0)949 return -EINVAL;950 951 /* To protect from overflow, divide by 1024 */952 stat->busy_time = load >> 10;953 stat->total_time = total >> 10;954 955 ret = exynos5_counters_set_event(dmc);956 if (ret < 0) {957 dev_err(dev, "could not set event counter\n");958 return ret;959 }960 }961 962 return 0;963}964 965/**966 * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency967 * @dev: device for which the framework checks operating frequency968 * @freq: returned frequency value969 *970 * It returns the currently used frequency of the DMC. The real operating971 * frequency might be lower when the clock source value could not be divided972 * to the requested value.973 */974static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)975{976 struct exynos5_dmc *dmc = dev_get_drvdata(dev);977 978 mutex_lock(&dmc->lock);979 *freq = dmc->curr_rate;980 mutex_unlock(&dmc->lock);981 982 return 0;983}984 985/*986 * exynos5_dmc_df_profile - Devfreq governor's profile structure987 *988 * It provides to the devfreq framework needed functions and polling period.989 */990static struct devfreq_dev_profile exynos5_dmc_df_profile = {991 .timer = DEVFREQ_TIMER_DELAYED,992 .target = exynos5_dmc_target,993 .get_dev_status = exynos5_dmc_get_status,994 .get_cur_freq = exynos5_dmc_get_cur_freq,995};996 997/**998 * exynos5_dmc_align_init_freq() - Align initial frequency value999 * @dmc: device for which the frequency is going to be set1000 * @bootloader_init_freq: initial frequency set by the bootloader in KHz1001 *1002 * The initial bootloader frequency, which is present during boot, might be1003 * different that supported frequency values in the driver. It is possible1004 * due to different PLL settings or used PLL as a source.1005 * This function provides the 'initial_freq' for the devfreq framework1006 * statistics engine which supports only registered values. Thus, some alignment1007 * must be made.1008 */1009static unsigned long1010exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,1011 unsigned long bootloader_init_freq)1012{1013 unsigned long aligned_freq;1014 int idx;1015 1016 idx = find_target_freq_idx(dmc, bootloader_init_freq);1017 if (idx >= 0)1018 aligned_freq = dmc->opp[idx].freq_hz;1019 else1020 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;1021 1022 return aligned_freq;1023}1024 1025/**1026 * create_timings_aligned() - Create register values and align with standard1027 * @dmc: device for which the frequency is going to be set1028 * @reg_timing_row: array to fill with values for timing row register1029 * @reg_timing_data: array to fill with values for timing data register1030 * @reg_timing_power: array to fill with values for timing power register1031 * @clk_period_ps: the period of the clock, known as tCK1032 *1033 * The function calculates timings and creates a register value ready for1034 * a frequency transition. The register contains a few timings. They are1035 * shifted by a known offset. The timing value is calculated based on memory1036 * specyfication: minimal time required and minimal cycles required.1037 */1038static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,1039 u32 *reg_timing_data, u32 *reg_timing_power,1040 u32 clk_period_ps)1041{1042 u32 val;1043 const struct timing_reg *reg;1044 1045 if (clk_period_ps == 0)1046 return -EINVAL;1047 1048 *reg_timing_row = 0;1049 *reg_timing_data = 0;1050 *reg_timing_power = 0;1051 1052 val = dmc->timings->tRFC / clk_period_ps;1053 val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;1054 val = max(val, dmc->min_tck->tRFC);1055 reg = &timing_row_reg_fields[0];1056 *reg_timing_row |= TIMING_VAL2REG(reg, val);1057 1058 val = dmc->timings->tRRD / clk_period_ps;1059 val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;1060 val = max(val, dmc->min_tck->tRRD);1061 reg = &timing_row_reg_fields[1];1062 *reg_timing_row |= TIMING_VAL2REG(reg, val);1063 1064 val = dmc->timings->tRPab / clk_period_ps;1065 val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;1066 val = max(val, dmc->min_tck->tRPab);1067 reg = &timing_row_reg_fields[2];1068 *reg_timing_row |= TIMING_VAL2REG(reg, val);1069 1070 val = dmc->timings->tRCD / clk_period_ps;1071 val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;1072 val = max(val, dmc->min_tck->tRCD);1073 reg = &timing_row_reg_fields[3];1074 *reg_timing_row |= TIMING_VAL2REG(reg, val);1075 1076 val = dmc->timings->tRC / clk_period_ps;1077 val += dmc->timings->tRC % clk_period_ps ? 1 : 0;1078 val = max(val, dmc->min_tck->tRC);1079 reg = &timing_row_reg_fields[4];1080 *reg_timing_row |= TIMING_VAL2REG(reg, val);1081 1082 val = dmc->timings->tRAS / clk_period_ps;1083 val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;1084 val = max(val, dmc->min_tck->tRAS);1085 reg = &timing_row_reg_fields[5];1086 *reg_timing_row |= TIMING_VAL2REG(reg, val);1087 1088 /* data related timings */1089 val = dmc->timings->tWTR / clk_period_ps;1090 val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;1091 val = max(val, dmc->min_tck->tWTR);1092 reg = &timing_data_reg_fields[0];1093 *reg_timing_data |= TIMING_VAL2REG(reg, val);1094 1095 val = dmc->timings->tWR / clk_period_ps;1096 val += dmc->timings->tWR % clk_period_ps ? 1 : 0;1097 val = max(val, dmc->min_tck->tWR);1098 reg = &timing_data_reg_fields[1];1099 *reg_timing_data |= TIMING_VAL2REG(reg, val);1100 1101 val = dmc->timings->tRTP / clk_period_ps;1102 val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;1103 val = max(val, dmc->min_tck->tRTP);1104 reg = &timing_data_reg_fields[2];1105 *reg_timing_data |= TIMING_VAL2REG(reg, val);1106 1107 val = dmc->timings->tW2W_C2C / clk_period_ps;1108 val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;1109 val = max(val, dmc->min_tck->tW2W_C2C);1110 reg = &timing_data_reg_fields[3];1111 *reg_timing_data |= TIMING_VAL2REG(reg, val);1112 1113 val = dmc->timings->tR2R_C2C / clk_period_ps;1114 val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;1115 val = max(val, dmc->min_tck->tR2R_C2C);1116 reg = &timing_data_reg_fields[4];1117 *reg_timing_data |= TIMING_VAL2REG(reg, val);1118 1119 val = dmc->timings->tWL / clk_period_ps;1120 val += dmc->timings->tWL % clk_period_ps ? 1 : 0;1121 val = max(val, dmc->min_tck->tWL);1122 reg = &timing_data_reg_fields[5];1123 *reg_timing_data |= TIMING_VAL2REG(reg, val);1124 1125 val = dmc->timings->tDQSCK / clk_period_ps;1126 val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;1127 val = max(val, dmc->min_tck->tDQSCK);1128 reg = &timing_data_reg_fields[6];1129 *reg_timing_data |= TIMING_VAL2REG(reg, val);1130 1131 val = dmc->timings->tRL / clk_period_ps;1132 val += dmc->timings->tRL % clk_period_ps ? 1 : 0;1133 val = max(val, dmc->min_tck->tRL);1134 reg = &timing_data_reg_fields[7];1135 *reg_timing_data |= TIMING_VAL2REG(reg, val);1136 1137 /* power related timings */1138 val = dmc->timings->tFAW / clk_period_ps;1139 val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;1140 val = max(val, dmc->min_tck->tFAW);1141 reg = &timing_power_reg_fields[0];1142 *reg_timing_power |= TIMING_VAL2REG(reg, val);1143 1144 val = dmc->timings->tXSR / clk_period_ps;1145 val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;1146 val = max(val, dmc->min_tck->tXSR);1147 reg = &timing_power_reg_fields[1];1148 *reg_timing_power |= TIMING_VAL2REG(reg, val);1149 1150 val = dmc->timings->tXP / clk_period_ps;1151 val += dmc->timings->tXP % clk_period_ps ? 1 : 0;1152 val = max(val, dmc->min_tck->tXP);1153 reg = &timing_power_reg_fields[2];1154 *reg_timing_power |= TIMING_VAL2REG(reg, val);1155 1156 val = dmc->timings->tCKE / clk_period_ps;1157 val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;1158 val = max(val, dmc->min_tck->tCKE);1159 reg = &timing_power_reg_fields[3];1160 *reg_timing_power |= TIMING_VAL2REG(reg, val);1161 1162 val = dmc->timings->tMRD / clk_period_ps;1163 val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;1164 val = max(val, dmc->min_tck->tMRD);1165 reg = &timing_power_reg_fields[4];1166 *reg_timing_power |= TIMING_VAL2REG(reg, val);1167 1168 return 0;1169}1170 1171/**1172 * of_get_dram_timings() - helper function for parsing DT settings for DRAM1173 * @dmc: device for which the frequency is going to be set1174 *1175 * The function parses DT entries with DRAM information.1176 */1177static int of_get_dram_timings(struct exynos5_dmc *dmc)1178{1179 int ret = 0;1180 struct device *dev = dmc->dev;1181 int idx;1182 u32 freq_mhz, clk_period_ps;1183 1184 struct device_node *np_ddr __free(device_node) =1185 of_parse_phandle(dev->of_node, "device-handle", 0);1186 if (!np_ddr) {1187 dev_warn(dev, "could not find 'device-handle' in DT\n");1188 return -EINVAL;1189 }1190 1191 dmc->timing_row = devm_kmalloc_array(dev, TIMING_COUNT,1192 sizeof(u32), GFP_KERNEL);1193 if (!dmc->timing_row)1194 return -ENOMEM;1195 1196 dmc->timing_data = devm_kmalloc_array(dev, TIMING_COUNT,1197 sizeof(u32), GFP_KERNEL);1198 if (!dmc->timing_data)1199 return -ENOMEM;1200 1201 dmc->timing_power = devm_kmalloc_array(dev, TIMING_COUNT,1202 sizeof(u32), GFP_KERNEL);1203 if (!dmc->timing_power)1204 return -ENOMEM;1205 1206 dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dev,1207 DDR_TYPE_LPDDR3,1208 &dmc->timings_arr_size);1209 if (!dmc->timings) {1210 dev_warn(dev, "could not get timings from DT\n");1211 return -EINVAL;1212 }1213 1214 dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dev);1215 if (!dmc->min_tck) {1216 dev_warn(dev, "could not get tck from DT\n");1217 return -EINVAL;1218 }1219 1220 /* Sorted array of OPPs with frequency ascending */1221 for (idx = 0; idx < dmc->opp_count; idx++) {1222 freq_mhz = dmc->opp[idx].freq_hz / 1000000;1223 clk_period_ps = 1000000 / freq_mhz;1224 1225 ret = create_timings_aligned(dmc, &dmc->timing_row[idx],1226 &dmc->timing_data[idx],1227 &dmc->timing_power[idx],1228 clk_period_ps);1229 }1230 1231 1232 /* Take the highest frequency's timings as 'bypass' */1233 dmc->bypass_timing_row = dmc->timing_row[idx - 1];1234 dmc->bypass_timing_data = dmc->timing_data[idx - 1];1235 dmc->bypass_timing_power = dmc->timing_power[idx - 1];1236 1237 return ret;1238}1239 1240/**1241 * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation.1242 * @dmc: DMC structure containing needed fields1243 *1244 * Get the needed clocks defined in DT device, enable and set the right parents.1245 * Read current frequency and initialize the initial rate for governor.1246 */1247static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)1248{1249 int ret;1250 struct device *dev = dmc->dev;1251 unsigned long target_volt = 0;1252 unsigned long target_rate = 0;1253 unsigned int tmp;1254 1255 dmc->fout_spll = devm_clk_get(dev, "fout_spll");1256 if (IS_ERR(dmc->fout_spll))1257 return PTR_ERR(dmc->fout_spll);1258 1259 dmc->fout_bpll = devm_clk_get(dev, "fout_bpll");1260 if (IS_ERR(dmc->fout_bpll))1261 return PTR_ERR(dmc->fout_bpll);1262 1263 dmc->mout_mclk_cdrex = devm_clk_get(dev, "mout_mclk_cdrex");1264 if (IS_ERR(dmc->mout_mclk_cdrex))1265 return PTR_ERR(dmc->mout_mclk_cdrex);1266 1267 dmc->mout_bpll = devm_clk_get(dev, "mout_bpll");1268 if (IS_ERR(dmc->mout_bpll))1269 return PTR_ERR(dmc->mout_bpll);1270 1271 dmc->mout_mx_mspll_ccore = devm_clk_get(dev, "mout_mx_mspll_ccore");1272 if (IS_ERR(dmc->mout_mx_mspll_ccore))1273 return PTR_ERR(dmc->mout_mx_mspll_ccore);1274 1275 dmc->mout_spll = devm_clk_get(dev, "ff_dout_spll2");1276 if (IS_ERR(dmc->mout_spll)) {1277 dmc->mout_spll = devm_clk_get(dev, "mout_sclk_spll");1278 if (IS_ERR(dmc->mout_spll))1279 return PTR_ERR(dmc->mout_spll);1280 }1281 1282 /*1283 * Convert frequency to KHz values and set it for the governor.1284 */1285 dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);1286 dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);1287 exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;1288 1289 ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,1290 &target_volt, 0);1291 if (ret)1292 return ret;1293 1294 dmc->curr_volt = target_volt;1295 1296 ret = clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);1297 if (ret)1298 return ret;1299 1300 clk_prepare_enable(dmc->fout_bpll);1301 clk_prepare_enable(dmc->mout_bpll);1302 1303 /*1304 * Some bootloaders do not set clock routes correctly.1305 * Stop one path in clocks to PHY.1306 */1307 regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);1308 tmp &= ~(BIT(1) | BIT(0));1309 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);1310 1311 return 0;1312}1313 1314/**1315 * exynos5_performance_counters_init() - Initializes performance DMC's counters1316 * @dmc: DMC for which it does the setup1317 *1318 * Initialization of performance counters in DMC for estimating usage.1319 * The counter's values are used for calculation of a memory bandwidth and based1320 * on that the governor changes the frequency.1321 * The counters are not used when the governor is GOVERNOR_USERSPACE.1322 */1323static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)1324{1325 struct device *dev = dmc->dev;1326 int ret, i;1327 1328 dmc->num_counters = devfreq_event_get_edev_count(dev, "devfreq-events");1329 if (dmc->num_counters < 0) {1330 dev_err(dev, "could not get devfreq-event counters\n");1331 return dmc->num_counters;1332 }1333 1334 dmc->counter = devm_kcalloc(dev, dmc->num_counters,1335 sizeof(*dmc->counter), GFP_KERNEL);1336 if (!dmc->counter)1337 return -ENOMEM;1338 1339 for (i = 0; i < dmc->num_counters; i++) {1340 dmc->counter[i] =1341 devfreq_event_get_edev_by_phandle(dev, "devfreq-events", i);1342 if (IS_ERR_OR_NULL(dmc->counter[i]))1343 return -EPROBE_DEFER;1344 }1345 1346 ret = exynos5_counters_enable_edev(dmc);1347 if (ret < 0) {1348 dev_err(dev, "could not enable event counter\n");1349 return ret;1350 }1351 1352 ret = exynos5_counters_set_event(dmc);1353 if (ret < 0) {1354 exynos5_counters_disable_edev(dmc);1355 dev_err(dev, "could not set event counter\n");1356 return ret;1357 }1358 1359 return 0;1360}1361 1362/**1363 * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC1364 * @dmc: device which is used for changing this feature1365 *1366 * There is a need of pausing DREX DMC when divider or MUX in clock tree1367 * changes its configuration. In such situation access to the memory is blocked1368 * in DMC automatically. This feature is used when clock frequency change1369 * request appears and touches clock tree.1370 */1371static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)1372{1373 unsigned int val;1374 int ret;1375 1376 ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);1377 if (ret)1378 return ret;1379 1380 val |= 1UL;1381 regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);1382 1383 return 0;1384}1385 1386static irqreturn_t dmc_irq_thread(int irq, void *priv)1387{1388 int res;1389 struct exynos5_dmc *dmc = priv;1390 1391 mutex_lock(&dmc->df->lock);1392 exynos5_dmc_perf_events_check(dmc);1393 res = update_devfreq(dmc->df);1394 mutex_unlock(&dmc->df->lock);1395 1396 if (res)1397 dev_warn(dmc->dev, "devfreq failed with %d\n", res);1398 1399 return IRQ_HANDLED;1400}1401 1402/**1403 * exynos5_dmc_probe() - Probe function for the DMC driver1404 * @pdev: platform device for which the driver is going to be initialized1405 *1406 * Initialize basic components: clocks, regulators, performance counters, etc.1407 * Read out product version and based on the information setup1408 * internal structures for the controller (frequency and voltage) and for DRAM1409 * memory parameters: timings for each operating frequency.1410 * Register new devfreq device for controlling DVFS of the DMC.1411 */1412static int exynos5_dmc_probe(struct platform_device *pdev)1413{1414 int ret = 0;1415 struct device *dev = &pdev->dev;1416 struct device_node *np = dev->of_node;1417 struct exynos5_dmc *dmc;1418 int irq[2];1419 1420 dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);1421 if (!dmc)1422 return -ENOMEM;1423 1424 mutex_init(&dmc->lock);1425 1426 dmc->dev = dev;1427 platform_set_drvdata(pdev, dmc);1428 1429 dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);1430 if (IS_ERR(dmc->base_drexi0))1431 return PTR_ERR(dmc->base_drexi0);1432 1433 dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);1434 if (IS_ERR(dmc->base_drexi1))1435 return PTR_ERR(dmc->base_drexi1);1436 1437 dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,1438 "samsung,syscon-clk");1439 if (IS_ERR(dmc->clk_regmap))1440 return PTR_ERR(dmc->clk_regmap);1441 1442 ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);1443 if (ret) {1444 dev_warn(dev, "couldn't initialize frequency settings\n");1445 return ret;1446 }1447 1448 dmc->vdd_mif = devm_regulator_get(dev, "vdd");1449 if (IS_ERR(dmc->vdd_mif)) {1450 ret = PTR_ERR(dmc->vdd_mif);1451 return ret;1452 }1453 1454 ret = exynos5_dmc_init_clks(dmc);1455 if (ret)1456 return ret;1457 1458 ret = of_get_dram_timings(dmc);1459 if (ret) {1460 dev_warn(dev, "couldn't initialize timings settings\n");1461 goto remove_clocks;1462 }1463 1464 ret = exynos5_dmc_set_pause_on_switching(dmc);1465 if (ret) {1466 dev_warn(dev, "couldn't get access to PAUSE register\n");1467 goto remove_clocks;1468 }1469 1470 /* There is two modes in which the driver works: polling or IRQ */1471 irq[0] = platform_get_irq_byname(pdev, "drex_0");1472 irq[1] = platform_get_irq_byname(pdev, "drex_1");1473 if (irq[0] > 0 && irq[1] > 0 && irqmode) {1474 ret = devm_request_threaded_irq(dev, irq[0], NULL,1475 dmc_irq_thread, IRQF_ONESHOT,1476 dev_name(dev), dmc);1477 if (ret) {1478 dev_err(dev, "couldn't grab IRQ\n");1479 goto remove_clocks;1480 }1481 1482 ret = devm_request_threaded_irq(dev, irq[1], NULL,1483 dmc_irq_thread, IRQF_ONESHOT,1484 dev_name(dev), dmc);1485 if (ret) {1486 dev_err(dev, "couldn't grab IRQ\n");1487 goto remove_clocks;1488 }1489 1490 /*1491 * Setup default thresholds for the devfreq governor.1492 * The values are chosen based on experiments.1493 */1494 dmc->gov_data.upthreshold = 55;1495 dmc->gov_data.downdifferential = 5;1496 1497 exynos5_dmc_enable_perf_events(dmc);1498 1499 dmc->in_irq_mode = 1;1500 } else {1501 ret = exynos5_performance_counters_init(dmc);1502 if (ret) {1503 dev_warn(dev, "couldn't probe performance counters\n");1504 goto remove_clocks;1505 }1506 1507 /*1508 * Setup default thresholds for the devfreq governor.1509 * The values are chosen based on experiments.1510 */1511 dmc->gov_data.upthreshold = 10;1512 dmc->gov_data.downdifferential = 5;1513 1514 exynos5_dmc_df_profile.polling_ms = 100;1515 }1516 1517 dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,1518 DEVFREQ_GOV_SIMPLE_ONDEMAND,1519 &dmc->gov_data);1520 1521 if (IS_ERR(dmc->df)) {1522 ret = PTR_ERR(dmc->df);1523 goto err_devfreq_add;1524 }1525 1526 if (dmc->in_irq_mode)1527 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);1528 1529 dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);1530 1531 return 0;1532 1533err_devfreq_add:1534 if (dmc->in_irq_mode)1535 exynos5_dmc_disable_perf_events(dmc);1536 else1537 exynos5_counters_disable_edev(dmc);1538remove_clocks:1539 clk_disable_unprepare(dmc->mout_bpll);1540 clk_disable_unprepare(dmc->fout_bpll);1541 1542 return ret;1543}1544 1545/**1546 * exynos5_dmc_remove() - Remove function for the platform device1547 * @pdev: platform device which is going to be removed1548 *1549 * The function relies on 'devm' framework function which automatically1550 * clean the device's resources. It just calls explicitly disable function for1551 * the performance counters.1552 */1553static void exynos5_dmc_remove(struct platform_device *pdev)1554{1555 struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);1556 1557 if (dmc->in_irq_mode)1558 exynos5_dmc_disable_perf_events(dmc);1559 else1560 exynos5_counters_disable_edev(dmc);1561 1562 clk_disable_unprepare(dmc->mout_bpll);1563 clk_disable_unprepare(dmc->fout_bpll);1564}1565 1566static const struct of_device_id exynos5_dmc_of_match[] = {1567 { .compatible = "samsung,exynos5422-dmc", },1568 { },1569};1570MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);1571 1572static struct platform_driver exynos5_dmc_platdrv = {1573 .probe = exynos5_dmc_probe,1574 .remove_new = exynos5_dmc_remove,1575 .driver = {1576 .name = "exynos5-dmc",1577 .of_match_table = exynos5_dmc_of_match,1578 },1579};1580module_platform_driver(exynos5_dmc_platdrv);1581MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");1582MODULE_LICENSE("GPL v2");1583MODULE_AUTHOR("Lukasz Luba");1584