1749 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * Tegra30 External Memory Controller driver4 *5 * Based on downstream driver from NVIDIA and tegra124-emc.c6 * Copyright (C) 2011-2014 NVIDIA Corporation7 *8 * Author: Dmitry Osipenko <digetx@gmail.com>9 * Copyright (C) 2019 GRATE-DRIVER project10 */11 12#include <linux/bitfield.h>13#include <linux/clk.h>14#include <linux/clk/tegra.h>15#include <linux/debugfs.h>16#include <linux/delay.h>17#include <linux/err.h>18#include <linux/interconnect-provider.h>19#include <linux/interrupt.h>20#include <linux/io.h>21#include <linux/iopoll.h>22#include <linux/kernel.h>23#include <linux/module.h>24#include <linux/mutex.h>25#include <linux/of.h>26#include <linux/platform_device.h>27#include <linux/pm_opp.h>28#include <linux/slab.h>29#include <linux/sort.h>30#include <linux/types.h>31 32#include <soc/tegra/common.h>33#include <soc/tegra/fuse.h>34 35#include "../jedec_ddr.h"36#include "../of_memory.h"37 38#include "mc.h"39 40#define EMC_INTSTATUS 0x00041#define EMC_INTMASK 0x00442#define EMC_DBG 0x00843#define EMC_ADR_CFG 0x01044#define EMC_CFG 0x00c45#define EMC_REFCTRL 0x02046#define EMC_TIMING_CONTROL 0x02847#define EMC_RC 0x02c48#define EMC_RFC 0x03049#define EMC_RAS 0x03450#define EMC_RP 0x03851#define EMC_R2W 0x03c52#define EMC_W2R 0x04053#define EMC_R2P 0x04454#define EMC_W2P 0x04855#define EMC_RD_RCD 0x04c56#define EMC_WR_RCD 0x05057#define EMC_RRD 0x05458#define EMC_REXT 0x05859#define EMC_WDV 0x05c60#define EMC_QUSE 0x06061#define EMC_QRST 0x06462#define EMC_QSAFE 0x06863#define EMC_RDV 0x06c64#define EMC_REFRESH 0x07065#define EMC_BURST_REFRESH_NUM 0x07466#define EMC_PDEX2WR 0x07867#define EMC_PDEX2RD 0x07c68#define EMC_PCHG2PDEN 0x08069#define EMC_ACT2PDEN 0x08470#define EMC_AR2PDEN 0x08871#define EMC_RW2PDEN 0x08c72#define EMC_TXSR 0x09073#define EMC_TCKE 0x09474#define EMC_TFAW 0x09875#define EMC_TRPAB 0x09c76#define EMC_TCLKSTABLE 0x0a077#define EMC_TCLKSTOP 0x0a478#define EMC_TREFBW 0x0a879#define EMC_QUSE_EXTRA 0x0ac80#define EMC_ODT_WRITE 0x0b081#define EMC_ODT_READ 0x0b482#define EMC_WEXT 0x0b883#define EMC_CTT 0x0bc84#define EMC_MRS_WAIT_CNT 0x0c885#define EMC_MRS 0x0cc86#define EMC_EMRS 0x0d087#define EMC_SELF_REF 0x0e088#define EMC_MRW 0x0e889#define EMC_MRR 0x0ec90#define EMC_XM2DQSPADCTRL3 0x0f891#define EMC_FBIO_SPARE 0x10092#define EMC_FBIO_CFG5 0x10493#define EMC_FBIO_CFG6 0x11494#define EMC_CFG_RSV 0x12095#define EMC_AUTO_CAL_CONFIG 0x2a496#define EMC_AUTO_CAL_INTERVAL 0x2a897#define EMC_AUTO_CAL_STATUS 0x2ac98#define EMC_STATUS 0x2b499#define EMC_CFG_2 0x2b8100#define EMC_CFG_DIG_DLL 0x2bc101#define EMC_CFG_DIG_DLL_PERIOD 0x2c0102#define EMC_CTT_DURATION 0x2d8103#define EMC_CTT_TERM_CTRL 0x2dc104#define EMC_ZCAL_INTERVAL 0x2e0105#define EMC_ZCAL_WAIT_CNT 0x2e4106#define EMC_ZQ_CAL 0x2ec107#define EMC_XM2CMDPADCTRL 0x2f0108#define EMC_XM2DQSPADCTRL2 0x2fc109#define EMC_XM2DQPADCTRL2 0x304110#define EMC_XM2CLKPADCTRL 0x308111#define EMC_XM2COMPPADCTRL 0x30c112#define EMC_XM2VTTGENPADCTRL 0x310113#define EMC_XM2VTTGENPADCTRL2 0x314114#define EMC_XM2QUSEPADCTRL 0x318115#define EMC_DLL_XFORM_DQS0 0x328116#define EMC_DLL_XFORM_DQS1 0x32c117#define EMC_DLL_XFORM_DQS2 0x330118#define EMC_DLL_XFORM_DQS3 0x334119#define EMC_DLL_XFORM_DQS4 0x338120#define EMC_DLL_XFORM_DQS5 0x33c121#define EMC_DLL_XFORM_DQS6 0x340122#define EMC_DLL_XFORM_DQS7 0x344123#define EMC_DLL_XFORM_QUSE0 0x348124#define EMC_DLL_XFORM_QUSE1 0x34c125#define EMC_DLL_XFORM_QUSE2 0x350126#define EMC_DLL_XFORM_QUSE3 0x354127#define EMC_DLL_XFORM_QUSE4 0x358128#define EMC_DLL_XFORM_QUSE5 0x35c129#define EMC_DLL_XFORM_QUSE6 0x360130#define EMC_DLL_XFORM_QUSE7 0x364131#define EMC_DLL_XFORM_DQ0 0x368132#define EMC_DLL_XFORM_DQ1 0x36c133#define EMC_DLL_XFORM_DQ2 0x370134#define EMC_DLL_XFORM_DQ3 0x374135#define EMC_DLI_TRIM_TXDQS0 0x3a8136#define EMC_DLI_TRIM_TXDQS1 0x3ac137#define EMC_DLI_TRIM_TXDQS2 0x3b0138#define EMC_DLI_TRIM_TXDQS3 0x3b4139#define EMC_DLI_TRIM_TXDQS4 0x3b8140#define EMC_DLI_TRIM_TXDQS5 0x3bc141#define EMC_DLI_TRIM_TXDQS6 0x3c0142#define EMC_DLI_TRIM_TXDQS7 0x3c4143#define EMC_STALL_THEN_EXE_BEFORE_CLKCHANGE 0x3c8144#define EMC_STALL_THEN_EXE_AFTER_CLKCHANGE 0x3cc145#define EMC_UNSTALL_RW_AFTER_CLKCHANGE 0x3d0146#define EMC_SEL_DPD_CTRL 0x3d8147#define EMC_PRE_REFRESH_REQ_CNT 0x3dc148#define EMC_DYN_SELF_REF_CONTROL 0x3e0149#define EMC_TXSRDLL 0x3e4150 151#define EMC_STATUS_TIMING_UPDATE_STALLED BIT(23)152 153#define EMC_MODE_SET_DLL_RESET BIT(8)154#define EMC_MODE_SET_LONG_CNT BIT(26)155 156#define EMC_SELF_REF_CMD_ENABLED BIT(0)157 158#define DRAM_DEV_SEL_ALL (0 << 30)159#define DRAM_DEV_SEL_0 BIT(31)160#define DRAM_DEV_SEL_1 BIT(30)161#define DRAM_BROADCAST(num) \162 ((num) > 1 ? DRAM_DEV_SEL_ALL : DRAM_DEV_SEL_0)163 164#define EMC_ZQ_CAL_CMD BIT(0)165#define EMC_ZQ_CAL_LONG BIT(4)166#define EMC_ZQ_CAL_LONG_CMD_DEV0 \167 (DRAM_DEV_SEL_0 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD)168#define EMC_ZQ_CAL_LONG_CMD_DEV1 \169 (DRAM_DEV_SEL_1 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD)170 171#define EMC_DBG_READ_MUX_ASSEMBLY BIT(0)172#define EMC_DBG_WRITE_MUX_ACTIVE BIT(1)173#define EMC_DBG_FORCE_UPDATE BIT(2)174#define EMC_DBG_CFG_PRIORITY BIT(24)175 176#define EMC_CFG5_QUSE_MODE_SHIFT 13177#define EMC_CFG5_QUSE_MODE_MASK (7 << EMC_CFG5_QUSE_MODE_SHIFT)178 179#define EMC_CFG5_QUSE_MODE_INTERNAL_LPBK 2180#define EMC_CFG5_QUSE_MODE_PULSE_INTERN 3181 182#define EMC_SEL_DPD_CTRL_QUSE_DPD_ENABLE BIT(9)183 184#define EMC_XM2COMPPADCTRL_VREF_CAL_ENABLE BIT(10)185 186#define EMC_XM2QUSEPADCTRL_IVREF_ENABLE BIT(4)187 188#define EMC_XM2DQSPADCTRL2_VREF_ENABLE BIT(5)189#define EMC_XM2DQSPADCTRL3_VREF_ENABLE BIT(5)190 191#define EMC_AUTO_CAL_STATUS_ACTIVE BIT(31)192 193#define EMC_FBIO_CFG5_DRAM_TYPE_MASK 0x3194 195#define EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK 0x3ff196#define EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT 16197#define EMC_MRS_WAIT_CNT_LONG_WAIT_MASK \198 (0x3ff << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT)199 200#define EMC_REFCTRL_DEV_SEL_MASK 0x3201#define EMC_REFCTRL_ENABLE BIT(31)202#define EMC_REFCTRL_ENABLE_ALL(num) \203 (((num) > 1 ? 0 : 2) | EMC_REFCTRL_ENABLE)204#define EMC_REFCTRL_DISABLE_ALL(num) ((num) > 1 ? 0 : 2)205 206#define EMC_CFG_PERIODIC_QRST BIT(21)207#define EMC_CFG_DYN_SREF_ENABLE BIT(28)208 209#define EMC_CLKCHANGE_REQ_ENABLE BIT(0)210#define EMC_CLKCHANGE_PD_ENABLE BIT(1)211#define EMC_CLKCHANGE_SR_ENABLE BIT(2)212 213#define EMC_TIMING_UPDATE BIT(0)214 215#define EMC_REFRESH_OVERFLOW_INT BIT(3)216#define EMC_CLKCHANGE_COMPLETE_INT BIT(4)217#define EMC_MRR_DIVLD_INT BIT(5)218 219#define EMC_MRR_DEV_SELECTN GENMASK(31, 30)220#define EMC_MRR_MRR_MA GENMASK(23, 16)221#define EMC_MRR_MRR_DATA GENMASK(15, 0)222 223#define EMC_ADR_CFG_EMEM_NUMDEV BIT(0)224 225enum emc_dram_type {226 DRAM_TYPE_DDR3,227 DRAM_TYPE_DDR1,228 DRAM_TYPE_LPDDR2,229 DRAM_TYPE_DDR2,230};231 232enum emc_dll_change {233 DLL_CHANGE_NONE,234 DLL_CHANGE_ON,235 DLL_CHANGE_OFF236};237 238static const u16 emc_timing_registers[] = {239 [0] = EMC_RC,240 [1] = EMC_RFC,241 [2] = EMC_RAS,242 [3] = EMC_RP,243 [4] = EMC_R2W,244 [5] = EMC_W2R,245 [6] = EMC_R2P,246 [7] = EMC_W2P,247 [8] = EMC_RD_RCD,248 [9] = EMC_WR_RCD,249 [10] = EMC_RRD,250 [11] = EMC_REXT,251 [12] = EMC_WEXT,252 [13] = EMC_WDV,253 [14] = EMC_QUSE,254 [15] = EMC_QRST,255 [16] = EMC_QSAFE,256 [17] = EMC_RDV,257 [18] = EMC_REFRESH,258 [19] = EMC_BURST_REFRESH_NUM,259 [20] = EMC_PRE_REFRESH_REQ_CNT,260 [21] = EMC_PDEX2WR,261 [22] = EMC_PDEX2RD,262 [23] = EMC_PCHG2PDEN,263 [24] = EMC_ACT2PDEN,264 [25] = EMC_AR2PDEN,265 [26] = EMC_RW2PDEN,266 [27] = EMC_TXSR,267 [28] = EMC_TXSRDLL,268 [29] = EMC_TCKE,269 [30] = EMC_TFAW,270 [31] = EMC_TRPAB,271 [32] = EMC_TCLKSTABLE,272 [33] = EMC_TCLKSTOP,273 [34] = EMC_TREFBW,274 [35] = EMC_QUSE_EXTRA,275 [36] = EMC_FBIO_CFG6,276 [37] = EMC_ODT_WRITE,277 [38] = EMC_ODT_READ,278 [39] = EMC_FBIO_CFG5,279 [40] = EMC_CFG_DIG_DLL,280 [41] = EMC_CFG_DIG_DLL_PERIOD,281 [42] = EMC_DLL_XFORM_DQS0,282 [43] = EMC_DLL_XFORM_DQS1,283 [44] = EMC_DLL_XFORM_DQS2,284 [45] = EMC_DLL_XFORM_DQS3,285 [46] = EMC_DLL_XFORM_DQS4,286 [47] = EMC_DLL_XFORM_DQS5,287 [48] = EMC_DLL_XFORM_DQS6,288 [49] = EMC_DLL_XFORM_DQS7,289 [50] = EMC_DLL_XFORM_QUSE0,290 [51] = EMC_DLL_XFORM_QUSE1,291 [52] = EMC_DLL_XFORM_QUSE2,292 [53] = EMC_DLL_XFORM_QUSE3,293 [54] = EMC_DLL_XFORM_QUSE4,294 [55] = EMC_DLL_XFORM_QUSE5,295 [56] = EMC_DLL_XFORM_QUSE6,296 [57] = EMC_DLL_XFORM_QUSE7,297 [58] = EMC_DLI_TRIM_TXDQS0,298 [59] = EMC_DLI_TRIM_TXDQS1,299 [60] = EMC_DLI_TRIM_TXDQS2,300 [61] = EMC_DLI_TRIM_TXDQS3,301 [62] = EMC_DLI_TRIM_TXDQS4,302 [63] = EMC_DLI_TRIM_TXDQS5,303 [64] = EMC_DLI_TRIM_TXDQS6,304 [65] = EMC_DLI_TRIM_TXDQS7,305 [66] = EMC_DLL_XFORM_DQ0,306 [67] = EMC_DLL_XFORM_DQ1,307 [68] = EMC_DLL_XFORM_DQ2,308 [69] = EMC_DLL_XFORM_DQ3,309 [70] = EMC_XM2CMDPADCTRL,310 [71] = EMC_XM2DQSPADCTRL2,311 [72] = EMC_XM2DQPADCTRL2,312 [73] = EMC_XM2CLKPADCTRL,313 [74] = EMC_XM2COMPPADCTRL,314 [75] = EMC_XM2VTTGENPADCTRL,315 [76] = EMC_XM2VTTGENPADCTRL2,316 [77] = EMC_XM2QUSEPADCTRL,317 [78] = EMC_XM2DQSPADCTRL3,318 [79] = EMC_CTT_TERM_CTRL,319 [80] = EMC_ZCAL_INTERVAL,320 [81] = EMC_ZCAL_WAIT_CNT,321 [82] = EMC_MRS_WAIT_CNT,322 [83] = EMC_AUTO_CAL_CONFIG,323 [84] = EMC_CTT,324 [85] = EMC_CTT_DURATION,325 [86] = EMC_DYN_SELF_REF_CONTROL,326 [87] = EMC_FBIO_SPARE,327 [88] = EMC_CFG_RSV,328};329 330struct emc_timing {331 unsigned long rate;332 333 u32 data[ARRAY_SIZE(emc_timing_registers)];334 335 u32 emc_auto_cal_interval;336 u32 emc_mode_1;337 u32 emc_mode_2;338 u32 emc_mode_reset;339 u32 emc_zcal_cnt_long;340 bool emc_cfg_periodic_qrst;341 bool emc_cfg_dyn_self_ref;342};343 344enum emc_rate_request_type {345 EMC_RATE_DEBUG,346 EMC_RATE_ICC,347 EMC_RATE_TYPE_MAX,348};349 350struct emc_rate_request {351 unsigned long min_rate;352 unsigned long max_rate;353};354 355struct tegra_emc {356 struct device *dev;357 struct tegra_mc *mc;358 struct icc_provider provider;359 struct notifier_block clk_nb;360 struct clk *clk;361 void __iomem *regs;362 unsigned int irq;363 bool bad_state;364 365 struct emc_timing *new_timing;366 struct emc_timing *timings;367 unsigned int num_timings;368 369 u32 mc_override;370 u32 emc_cfg;371 372 u32 emc_mode_1;373 u32 emc_mode_2;374 u32 emc_mode_reset;375 376 bool vref_cal_toggle : 1;377 bool zcal_long : 1;378 bool dll_on : 1;379 380 struct {381 struct dentry *root;382 unsigned long min_rate;383 unsigned long max_rate;384 } debugfs;385 386 /*387 * There are multiple sources in the EMC driver which could request388 * a min/max clock rate, these rates are contained in this array.389 */390 struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX];391 392 /* protect shared rate-change code path */393 struct mutex rate_lock;394 395 bool mrr_error;396};397 398static int emc_seq_update_timing(struct tegra_emc *emc)399{400 u32 val;401 int err;402 403 writel_relaxed(EMC_TIMING_UPDATE, emc->regs + EMC_TIMING_CONTROL);404 405 err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_STATUS, val,406 !(val & EMC_STATUS_TIMING_UPDATE_STALLED),407 1, 200);408 if (err) {409 dev_err(emc->dev, "failed to update timing: %d\n", err);410 return err;411 }412 413 return 0;414}415 416static irqreturn_t tegra_emc_isr(int irq, void *data)417{418 struct tegra_emc *emc = data;419 u32 intmask = EMC_REFRESH_OVERFLOW_INT;420 u32 status;421 422 status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask;423 if (!status)424 return IRQ_NONE;425 426 /* notify about HW problem */427 if (status & EMC_REFRESH_OVERFLOW_INT)428 dev_err_ratelimited(emc->dev,429 "refresh request overflow timeout\n");430 431 /* clear interrupts */432 writel_relaxed(status, emc->regs + EMC_INTSTATUS);433 434 return IRQ_HANDLED;435}436 437static struct emc_timing *emc_find_timing(struct tegra_emc *emc,438 unsigned long rate)439{440 struct emc_timing *timing = NULL;441 unsigned int i;442 443 for (i = 0; i < emc->num_timings; i++) {444 if (emc->timings[i].rate >= rate) {445 timing = &emc->timings[i];446 break;447 }448 }449 450 if (!timing) {451 dev_err(emc->dev, "no timing for rate %lu\n", rate);452 return NULL;453 }454 455 return timing;456}457 458static bool emc_dqs_preset(struct tegra_emc *emc, struct emc_timing *timing,459 bool *schmitt_to_vref)460{461 bool preset = false;462 u32 val;463 464 if (timing->data[71] & EMC_XM2DQSPADCTRL2_VREF_ENABLE) {465 val = readl_relaxed(emc->regs + EMC_XM2DQSPADCTRL2);466 467 if (!(val & EMC_XM2DQSPADCTRL2_VREF_ENABLE)) {468 val |= EMC_XM2DQSPADCTRL2_VREF_ENABLE;469 writel_relaxed(val, emc->regs + EMC_XM2DQSPADCTRL2);470 471 preset = true;472 }473 }474 475 if (timing->data[78] & EMC_XM2DQSPADCTRL3_VREF_ENABLE) {476 val = readl_relaxed(emc->regs + EMC_XM2DQSPADCTRL3);477 478 if (!(val & EMC_XM2DQSPADCTRL3_VREF_ENABLE)) {479 val |= EMC_XM2DQSPADCTRL3_VREF_ENABLE;480 writel_relaxed(val, emc->regs + EMC_XM2DQSPADCTRL3);481 482 preset = true;483 }484 }485 486 if (timing->data[77] & EMC_XM2QUSEPADCTRL_IVREF_ENABLE) {487 val = readl_relaxed(emc->regs + EMC_XM2QUSEPADCTRL);488 489 if (!(val & EMC_XM2QUSEPADCTRL_IVREF_ENABLE)) {490 val |= EMC_XM2QUSEPADCTRL_IVREF_ENABLE;491 writel_relaxed(val, emc->regs + EMC_XM2QUSEPADCTRL);492 493 *schmitt_to_vref = true;494 preset = true;495 }496 }497 498 return preset;499}500 501static int emc_prepare_mc_clk_cfg(struct tegra_emc *emc, unsigned long rate)502{503 struct tegra_mc *mc = emc->mc;504 unsigned int misc0_index = 16;505 unsigned int i;506 bool same;507 508 for (i = 0; i < mc->num_timings; i++) {509 if (mc->timings[i].rate != rate)510 continue;511 512 if (mc->timings[i].emem_data[misc0_index] & BIT(27))513 same = true;514 else515 same = false;516 517 return tegra20_clk_prepare_emc_mc_same_freq(emc->clk, same);518 }519 520 return -EINVAL;521}522 523static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate)524{525 struct emc_timing *timing = emc_find_timing(emc, rate);526 enum emc_dll_change dll_change;527 enum emc_dram_type dram_type;528 bool schmitt_to_vref = false;529 unsigned int pre_wait = 0;530 bool qrst_used = false;531 unsigned int dram_num;532 unsigned int i;533 u32 fbio_cfg5;534 u32 emc_dbg;535 u32 val;536 int err;537 538 if (!timing || emc->bad_state)539 return -EINVAL;540 541 dev_dbg(emc->dev, "%s: using timing rate %lu for requested rate %lu\n",542 __func__, timing->rate, rate);543 544 emc->bad_state = true;545 546 err = emc_prepare_mc_clk_cfg(emc, rate);547 if (err) {548 dev_err(emc->dev, "mc clock preparation failed: %d\n", err);549 return err;550 }551 552 emc->vref_cal_toggle = false;553 emc->mc_override = mc_readl(emc->mc, MC_EMEM_ARB_OVERRIDE);554 emc->emc_cfg = readl_relaxed(emc->regs + EMC_CFG);555 emc_dbg = readl_relaxed(emc->regs + EMC_DBG);556 557 if (emc->dll_on == !!(timing->emc_mode_1 & 0x1))558 dll_change = DLL_CHANGE_NONE;559 else if (timing->emc_mode_1 & 0x1)560 dll_change = DLL_CHANGE_ON;561 else562 dll_change = DLL_CHANGE_OFF;563 564 emc->dll_on = !!(timing->emc_mode_1 & 0x1);565 566 if (timing->data[80] && !readl_relaxed(emc->regs + EMC_ZCAL_INTERVAL))567 emc->zcal_long = true;568 else569 emc->zcal_long = false;570 571 fbio_cfg5 = readl_relaxed(emc->regs + EMC_FBIO_CFG5);572 dram_type = fbio_cfg5 & EMC_FBIO_CFG5_DRAM_TYPE_MASK;573 574 dram_num = tegra_mc_get_emem_device_count(emc->mc);575 576 /* disable dynamic self-refresh */577 if (emc->emc_cfg & EMC_CFG_DYN_SREF_ENABLE) {578 emc->emc_cfg &= ~EMC_CFG_DYN_SREF_ENABLE;579 writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG);580 581 pre_wait = 5;582 }583 584 /* update MC arbiter settings */585 val = mc_readl(emc->mc, MC_EMEM_ARB_OUTSTANDING_REQ);586 if (!(val & MC_EMEM_ARB_OUTSTANDING_REQ_HOLDOFF_OVERRIDE) ||587 ((val & MC_EMEM_ARB_OUTSTANDING_REQ_MAX_MASK) > 0x50)) {588 589 val = MC_EMEM_ARB_OUTSTANDING_REQ_LIMIT_ENABLE |590 MC_EMEM_ARB_OUTSTANDING_REQ_HOLDOFF_OVERRIDE | 0x50;591 mc_writel(emc->mc, val, MC_EMEM_ARB_OUTSTANDING_REQ);592 mc_writel(emc->mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);593 }594 595 if (emc->mc_override & MC_EMEM_ARB_OVERRIDE_EACK_MASK)596 mc_writel(emc->mc,597 emc->mc_override & ~MC_EMEM_ARB_OVERRIDE_EACK_MASK,598 MC_EMEM_ARB_OVERRIDE);599 600 /* check DQ/DQS VREF delay */601 if (emc_dqs_preset(emc, timing, &schmitt_to_vref)) {602 if (pre_wait < 3)603 pre_wait = 3;604 }605 606 if (pre_wait) {607 err = emc_seq_update_timing(emc);608 if (err)609 return err;610 611 udelay(pre_wait);612 }613 614 /* disable auto-calibration if VREF mode is switching */615 if (timing->emc_auto_cal_interval) {616 val = readl_relaxed(emc->regs + EMC_XM2COMPPADCTRL);617 val ^= timing->data[74];618 619 if (val & EMC_XM2COMPPADCTRL_VREF_CAL_ENABLE) {620 writel_relaxed(0, emc->regs + EMC_AUTO_CAL_INTERVAL);621 622 err = readl_relaxed_poll_timeout_atomic(623 emc->regs + EMC_AUTO_CAL_STATUS, val,624 !(val & EMC_AUTO_CAL_STATUS_ACTIVE), 1, 300);625 if (err) {626 dev_err(emc->dev,627 "auto-cal finish timeout: %d\n", err);628 return err;629 }630 631 emc->vref_cal_toggle = true;632 }633 }634 635 /* program shadow registers */636 for (i = 0; i < ARRAY_SIZE(timing->data); i++) {637 /* EMC_XM2CLKPADCTRL should be programmed separately */638 if (i != 73)639 writel_relaxed(timing->data[i],640 emc->regs + emc_timing_registers[i]);641 }642 643 err = tegra_mc_write_emem_configuration(emc->mc, timing->rate);644 if (err)645 return err;646 647 /* DDR3: predict MRS long wait count */648 if (dram_type == DRAM_TYPE_DDR3 && dll_change == DLL_CHANGE_ON) {649 u32 cnt = 512;650 651 if (emc->zcal_long)652 cnt -= dram_num * 256;653 654 val = timing->data[82] & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK;655 if (cnt < val)656 cnt = val;657 658 val = timing->data[82] & ~EMC_MRS_WAIT_CNT_LONG_WAIT_MASK;659 val |= (cnt << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT) &660 EMC_MRS_WAIT_CNT_LONG_WAIT_MASK;661 662 writel_relaxed(val, emc->regs + EMC_MRS_WAIT_CNT);663 }664 665 /* this read also completes the writes */666 val = readl_relaxed(emc->regs + EMC_SEL_DPD_CTRL);667 668 if (!(val & EMC_SEL_DPD_CTRL_QUSE_DPD_ENABLE) && schmitt_to_vref) {669 u32 cur_mode, new_mode;670 671 cur_mode = fbio_cfg5 & EMC_CFG5_QUSE_MODE_MASK;672 cur_mode >>= EMC_CFG5_QUSE_MODE_SHIFT;673 674 new_mode = timing->data[39] & EMC_CFG5_QUSE_MODE_MASK;675 new_mode >>= EMC_CFG5_QUSE_MODE_SHIFT;676 677 if ((cur_mode != EMC_CFG5_QUSE_MODE_PULSE_INTERN &&678 cur_mode != EMC_CFG5_QUSE_MODE_INTERNAL_LPBK) ||679 (new_mode != EMC_CFG5_QUSE_MODE_PULSE_INTERN &&680 new_mode != EMC_CFG5_QUSE_MODE_INTERNAL_LPBK))681 qrst_used = true;682 }683 684 /* flow control marker 1 */685 writel_relaxed(0x1, emc->regs + EMC_STALL_THEN_EXE_BEFORE_CLKCHANGE);686 687 /* enable periodic reset */688 if (qrst_used) {689 writel_relaxed(emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE,690 emc->regs + EMC_DBG);691 writel_relaxed(emc->emc_cfg | EMC_CFG_PERIODIC_QRST,692 emc->regs + EMC_CFG);693 writel_relaxed(emc_dbg, emc->regs + EMC_DBG);694 }695 696 /* disable auto-refresh to save time after clock change */697 writel_relaxed(EMC_REFCTRL_DISABLE_ALL(dram_num),698 emc->regs + EMC_REFCTRL);699 700 /* turn off DLL and enter self-refresh on DDR3 */701 if (dram_type == DRAM_TYPE_DDR3) {702 if (dll_change == DLL_CHANGE_OFF)703 writel_relaxed(timing->emc_mode_1,704 emc->regs + EMC_EMRS);705 706 writel_relaxed(DRAM_BROADCAST(dram_num) |707 EMC_SELF_REF_CMD_ENABLED,708 emc->regs + EMC_SELF_REF);709 }710 711 /* flow control marker 2 */712 writel_relaxed(0x1, emc->regs + EMC_STALL_THEN_EXE_AFTER_CLKCHANGE);713 714 /* enable write-active MUX, update unshadowed pad control */715 writel_relaxed(emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, emc->regs + EMC_DBG);716 writel_relaxed(timing->data[73], emc->regs + EMC_XM2CLKPADCTRL);717 718 /* restore periodic QRST and disable write-active MUX */719 val = !!(emc->emc_cfg & EMC_CFG_PERIODIC_QRST);720 if (qrst_used || timing->emc_cfg_periodic_qrst != val) {721 if (timing->emc_cfg_periodic_qrst)722 emc->emc_cfg |= EMC_CFG_PERIODIC_QRST;723 else724 emc->emc_cfg &= ~EMC_CFG_PERIODIC_QRST;725 726 writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG);727 }728 writel_relaxed(emc_dbg, emc->regs + EMC_DBG);729 730 /* exit self-refresh on DDR3 */731 if (dram_type == DRAM_TYPE_DDR3)732 writel_relaxed(DRAM_BROADCAST(dram_num),733 emc->regs + EMC_SELF_REF);734 735 /* set DRAM-mode registers */736 if (dram_type == DRAM_TYPE_DDR3) {737 if (timing->emc_mode_1 != emc->emc_mode_1)738 writel_relaxed(timing->emc_mode_1,739 emc->regs + EMC_EMRS);740 741 if (timing->emc_mode_2 != emc->emc_mode_2)742 writel_relaxed(timing->emc_mode_2,743 emc->regs + EMC_EMRS);744 745 if (timing->emc_mode_reset != emc->emc_mode_reset ||746 dll_change == DLL_CHANGE_ON) {747 val = timing->emc_mode_reset;748 if (dll_change == DLL_CHANGE_ON) {749 val |= EMC_MODE_SET_DLL_RESET;750 val |= EMC_MODE_SET_LONG_CNT;751 } else {752 val &= ~EMC_MODE_SET_DLL_RESET;753 }754 writel_relaxed(val, emc->regs + EMC_MRS);755 }756 } else {757 if (timing->emc_mode_2 != emc->emc_mode_2)758 writel_relaxed(timing->emc_mode_2,759 emc->regs + EMC_MRW);760 761 if (timing->emc_mode_1 != emc->emc_mode_1)762 writel_relaxed(timing->emc_mode_1,763 emc->regs + EMC_MRW);764 }765 766 emc->emc_mode_1 = timing->emc_mode_1;767 emc->emc_mode_2 = timing->emc_mode_2;768 emc->emc_mode_reset = timing->emc_mode_reset;769 770 /* issue ZCAL command if turning ZCAL on */771 if (emc->zcal_long) {772 writel_relaxed(EMC_ZQ_CAL_LONG_CMD_DEV0,773 emc->regs + EMC_ZQ_CAL);774 775 if (dram_num > 1)776 writel_relaxed(EMC_ZQ_CAL_LONG_CMD_DEV1,777 emc->regs + EMC_ZQ_CAL);778 }779 780 /* flow control marker 3 */781 writel_relaxed(0x1, emc->regs + EMC_UNSTALL_RW_AFTER_CLKCHANGE);782 783 /*784 * Read and discard an arbitrary MC register (Note: EMC registers785 * can't be used) to ensure the register writes are completed.786 */787 mc_readl(emc->mc, MC_EMEM_ARB_OVERRIDE);788 789 return 0;790}791 792static int emc_complete_timing_change(struct tegra_emc *emc,793 unsigned long rate)794{795 struct emc_timing *timing = emc_find_timing(emc, rate);796 unsigned int dram_num;797 int err;798 u32 v;799 800 err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, v,801 v & EMC_CLKCHANGE_COMPLETE_INT,802 1, 100);803 if (err) {804 dev_err(emc->dev, "emc-car handshake timeout: %d\n", err);805 return err;806 }807 808 /* re-enable auto-refresh */809 dram_num = tegra_mc_get_emem_device_count(emc->mc);810 writel_relaxed(EMC_REFCTRL_ENABLE_ALL(dram_num),811 emc->regs + EMC_REFCTRL);812 813 /* restore auto-calibration */814 if (emc->vref_cal_toggle)815 writel_relaxed(timing->emc_auto_cal_interval,816 emc->regs + EMC_AUTO_CAL_INTERVAL);817 818 /* restore dynamic self-refresh */819 if (timing->emc_cfg_dyn_self_ref) {820 emc->emc_cfg |= EMC_CFG_DYN_SREF_ENABLE;821 writel_relaxed(emc->emc_cfg, emc->regs + EMC_CFG);822 }823 824 /* set number of clocks to wait after each ZQ command */825 if (emc->zcal_long)826 writel_relaxed(timing->emc_zcal_cnt_long,827 emc->regs + EMC_ZCAL_WAIT_CNT);828 829 /* wait for writes to settle */830 udelay(2);831 832 /* update restored timing */833 err = emc_seq_update_timing(emc);834 if (!err)835 emc->bad_state = false;836 837 /* restore early ACK */838 mc_writel(emc->mc, emc->mc_override, MC_EMEM_ARB_OVERRIDE);839 840 return err;841}842 843static int emc_unprepare_timing_change(struct tegra_emc *emc,844 unsigned long rate)845{846 if (!emc->bad_state) {847 /* shouldn't ever happen in practice */848 dev_err(emc->dev, "timing configuration can't be reverted\n");849 emc->bad_state = true;850 }851 852 return 0;853}854 855static int emc_clk_change_notify(struct notifier_block *nb,856 unsigned long msg, void *data)857{858 struct tegra_emc *emc = container_of(nb, struct tegra_emc, clk_nb);859 struct clk_notifier_data *cnd = data;860 int err;861 862 switch (msg) {863 case PRE_RATE_CHANGE:864 /*865 * Disable interrupt since read accesses are prohibited after866 * stalling.867 */868 disable_irq(emc->irq);869 err = emc_prepare_timing_change(emc, cnd->new_rate);870 enable_irq(emc->irq);871 break;872 873 case ABORT_RATE_CHANGE:874 err = emc_unprepare_timing_change(emc, cnd->old_rate);875 break;876 877 case POST_RATE_CHANGE:878 err = emc_complete_timing_change(emc, cnd->new_rate);879 break;880 881 default:882 return NOTIFY_DONE;883 }884 885 return notifier_from_errno(err);886}887 888static int load_one_timing_from_dt(struct tegra_emc *emc,889 struct emc_timing *timing,890 struct device_node *node)891{892 u32 value;893 int err;894 895 err = of_property_read_u32(node, "clock-frequency", &value);896 if (err) {897 dev_err(emc->dev, "timing %pOF: failed to read rate: %d\n",898 node, err);899 return err;900 }901 902 timing->rate = value;903 904 err = of_property_read_u32_array(node, "nvidia,emc-configuration",905 timing->data,906 ARRAY_SIZE(emc_timing_registers));907 if (err) {908 dev_err(emc->dev,909 "timing %pOF: failed to read emc timing data: %d\n",910 node, err);911 return err;912 }913 914#define EMC_READ_BOOL(prop, dtprop) \915 timing->prop = of_property_read_bool(node, dtprop);916 917#define EMC_READ_U32(prop, dtprop) \918 err = of_property_read_u32(node, dtprop, &timing->prop); \919 if (err) { \920 dev_err(emc->dev, \921 "timing %pOFn: failed to read " #prop ": %d\n", \922 node, err); \923 return err; \924 }925 926 EMC_READ_U32(emc_auto_cal_interval, "nvidia,emc-auto-cal-interval")927 EMC_READ_U32(emc_mode_1, "nvidia,emc-mode-1")928 EMC_READ_U32(emc_mode_2, "nvidia,emc-mode-2")929 EMC_READ_U32(emc_mode_reset, "nvidia,emc-mode-reset")930 EMC_READ_U32(emc_zcal_cnt_long, "nvidia,emc-zcal-cnt-long")931 EMC_READ_BOOL(emc_cfg_dyn_self_ref, "nvidia,emc-cfg-dyn-self-ref")932 EMC_READ_BOOL(emc_cfg_periodic_qrst, "nvidia,emc-cfg-periodic-qrst")933 934#undef EMC_READ_U32935#undef EMC_READ_BOOL936 937 dev_dbg(emc->dev, "%s: %pOF: rate %lu\n", __func__, node, timing->rate);938 939 return 0;940}941 942static int cmp_timings(const void *_a, const void *_b)943{944 const struct emc_timing *a = _a;945 const struct emc_timing *b = _b;946 947 if (a->rate < b->rate)948 return -1;949 950 if (a->rate > b->rate)951 return 1;952 953 return 0;954}955 956static int emc_check_mc_timings(struct tegra_emc *emc)957{958 struct tegra_mc *mc = emc->mc;959 unsigned int i;960 961 if (emc->num_timings != mc->num_timings) {962 dev_err(emc->dev, "emc/mc timings number mismatch: %u %u\n",963 emc->num_timings, mc->num_timings);964 return -EINVAL;965 }966 967 for (i = 0; i < mc->num_timings; i++) {968 if (emc->timings[i].rate != mc->timings[i].rate) {969 dev_err(emc->dev,970 "emc/mc timing rate mismatch: %lu %lu\n",971 emc->timings[i].rate, mc->timings[i].rate);972 return -EINVAL;973 }974 }975 976 return 0;977}978 979static int emc_load_timings_from_dt(struct tegra_emc *emc,980 struct device_node *node)981{982 struct emc_timing *timing;983 int child_count;984 int err;985 986 child_count = of_get_child_count(node);987 if (!child_count) {988 dev_err(emc->dev, "no memory timings in: %pOF\n", node);989 return -EINVAL;990 }991 992 emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing),993 GFP_KERNEL);994 if (!emc->timings)995 return -ENOMEM;996 997 emc->num_timings = child_count;998 timing = emc->timings;999 1000 for_each_child_of_node_scoped(node, child) {1001 err = load_one_timing_from_dt(emc, timing++, child);1002 if (err)1003 return err;1004 }1005 1006 sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings,1007 NULL);1008 1009 err = emc_check_mc_timings(emc);1010 if (err)1011 return err;1012 1013 dev_info_once(emc->dev,1014 "got %u timings for RAM code %u (min %luMHz max %luMHz)\n",1015 emc->num_timings,1016 tegra_read_ram_code(),1017 emc->timings[0].rate / 1000000,1018 emc->timings[emc->num_timings - 1].rate / 1000000);1019 1020 return 0;1021}1022 1023static struct device_node *emc_find_node_by_ram_code(struct tegra_emc *emc)1024{1025 struct device *dev = emc->dev;1026 struct device_node *np;1027 u32 value, ram_code;1028 int err;1029 1030 if (emc->mrr_error) {1031 dev_warn(dev, "memory timings skipped due to MRR error\n");1032 return NULL;1033 }1034 1035 if (of_get_child_count(dev->of_node) == 0) {1036 dev_info_once(dev, "device-tree doesn't have memory timings\n");1037 return NULL;1038 }1039 1040 ram_code = tegra_read_ram_code();1041 1042 for_each_child_of_node(dev->of_node, np) {1043 err = of_property_read_u32(np, "nvidia,ram-code", &value);1044 if (err || value != ram_code)1045 continue;1046 1047 return np;1048 }1049 1050 dev_err(dev, "no memory timings for RAM code %u found in device-tree\n",1051 ram_code);1052 1053 return NULL;1054}1055 1056static int emc_read_lpddr_mode_register(struct tegra_emc *emc,1057 unsigned int emem_dev,1058 unsigned int register_addr,1059 unsigned int *register_data)1060{1061 u32 memory_dev = emem_dev ? 1 : 2;1062 u32 val, mr_mask = 0xff;1063 int err;1064 1065 /* clear data-valid interrupt status */1066 writel_relaxed(EMC_MRR_DIVLD_INT, emc->regs + EMC_INTSTATUS);1067 1068 /* issue mode register read request */1069 val = FIELD_PREP(EMC_MRR_DEV_SELECTN, memory_dev);1070 val |= FIELD_PREP(EMC_MRR_MRR_MA, register_addr);1071 1072 writel_relaxed(val, emc->regs + EMC_MRR);1073 1074 /* wait for the LPDDR2 data-valid interrupt */1075 err = readl_relaxed_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, val,1076 val & EMC_MRR_DIVLD_INT,1077 1, 100);1078 if (err) {1079 dev_err(emc->dev, "mode register %u read failed: %d\n",1080 register_addr, err);1081 emc->mrr_error = true;1082 return err;1083 }1084 1085 /* read out mode register data */1086 val = readl_relaxed(emc->regs + EMC_MRR);1087 *register_data = FIELD_GET(EMC_MRR_MRR_DATA, val) & mr_mask;1088 1089 return 0;1090}1091 1092static void emc_read_lpddr_sdram_info(struct tegra_emc *emc,1093 unsigned int emem_dev)1094{1095 union lpddr2_basic_config4 basic_conf4;1096 unsigned int manufacturer_id;1097 unsigned int revision_id1;1098 unsigned int revision_id2;1099 1100 /* these registers are standard for all LPDDR JEDEC memory chips */1101 emc_read_lpddr_mode_register(emc, emem_dev, 5, &manufacturer_id);1102 emc_read_lpddr_mode_register(emc, emem_dev, 6, &revision_id1);1103 emc_read_lpddr_mode_register(emc, emem_dev, 7, &revision_id2);1104 emc_read_lpddr_mode_register(emc, emem_dev, 8, &basic_conf4.value);1105 1106 dev_info(emc->dev, "SDRAM[dev%u]: manufacturer: 0x%x (%s) rev1: 0x%x rev2: 0x%x prefetch: S%u density: %uMbit iowidth: %ubit\n",1107 emem_dev, manufacturer_id,1108 lpddr2_jedec_manufacturer(manufacturer_id),1109 revision_id1, revision_id2,1110 4 >> basic_conf4.arch_type,1111 64 << basic_conf4.density,1112 32 >> basic_conf4.io_width);1113}1114 1115static int emc_setup_hw(struct tegra_emc *emc)1116{1117 u32 fbio_cfg5, emc_cfg, emc_dbg, emc_adr_cfg;1118 u32 intmask = EMC_REFRESH_OVERFLOW_INT;1119 static bool print_sdram_info_once;1120 enum emc_dram_type dram_type;1121 const char *dram_type_str;1122 unsigned int emem_numdev;1123 1124 fbio_cfg5 = readl_relaxed(emc->regs + EMC_FBIO_CFG5);1125 dram_type = fbio_cfg5 & EMC_FBIO_CFG5_DRAM_TYPE_MASK;1126 1127 emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2);1128 1129 /* enable EMC and CAR to handshake on PLL divider/source changes */1130 emc_cfg |= EMC_CLKCHANGE_REQ_ENABLE;1131 1132 /* configure clock change mode accordingly to DRAM type */1133 switch (dram_type) {1134 case DRAM_TYPE_LPDDR2:1135 emc_cfg |= EMC_CLKCHANGE_PD_ENABLE;1136 emc_cfg &= ~EMC_CLKCHANGE_SR_ENABLE;1137 break;1138 1139 default:1140 emc_cfg &= ~EMC_CLKCHANGE_SR_ENABLE;1141 emc_cfg &= ~EMC_CLKCHANGE_PD_ENABLE;1142 break;1143 }1144 1145 writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2);1146 1147 /* initialize interrupt */1148 writel_relaxed(intmask, emc->regs + EMC_INTMASK);1149 writel_relaxed(0xffffffff, emc->regs + EMC_INTSTATUS);1150 1151 /* ensure that unwanted debug features are disabled */1152 emc_dbg = readl_relaxed(emc->regs + EMC_DBG);1153 emc_dbg |= EMC_DBG_CFG_PRIORITY;1154 emc_dbg &= ~EMC_DBG_READ_MUX_ASSEMBLY;1155 emc_dbg &= ~EMC_DBG_WRITE_MUX_ACTIVE;1156 emc_dbg &= ~EMC_DBG_FORCE_UPDATE;1157 writel_relaxed(emc_dbg, emc->regs + EMC_DBG);1158 1159 switch (dram_type) {1160 case DRAM_TYPE_DDR1:1161 dram_type_str = "DDR1";1162 break;1163 case DRAM_TYPE_LPDDR2:1164 dram_type_str = "LPDDR2";1165 break;1166 case DRAM_TYPE_DDR2:1167 dram_type_str = "DDR2";1168 break;1169 case DRAM_TYPE_DDR3:1170 dram_type_str = "DDR3";1171 break;1172 }1173 1174 emc_adr_cfg = readl_relaxed(emc->regs + EMC_ADR_CFG);1175 emem_numdev = FIELD_GET(EMC_ADR_CFG_EMEM_NUMDEV, emc_adr_cfg) + 1;1176 1177 dev_info_once(emc->dev, "%u %s %s attached\n", emem_numdev,1178 dram_type_str, emem_numdev == 2 ? "devices" : "device");1179 1180 if (dram_type == DRAM_TYPE_LPDDR2 && !print_sdram_info_once) {1181 while (emem_numdev--)1182 emc_read_lpddr_sdram_info(emc, emem_numdev);1183 1184 print_sdram_info_once = true;1185 }1186 1187 return 0;1188}1189 1190static long emc_round_rate(unsigned long rate,1191 unsigned long min_rate,1192 unsigned long max_rate,1193 void *arg)1194{1195 struct emc_timing *timing = NULL;1196 struct tegra_emc *emc = arg;1197 unsigned int i;1198 1199 if (!emc->num_timings)1200 return clk_get_rate(emc->clk);1201 1202 min_rate = min(min_rate, emc->timings[emc->num_timings - 1].rate);1203 1204 for (i = 0; i < emc->num_timings; i++) {1205 if (emc->timings[i].rate < rate && i != emc->num_timings - 1)1206 continue;1207 1208 if (emc->timings[i].rate > max_rate) {1209 i = max(i, 1u) - 1;1210 1211 if (emc->timings[i].rate < min_rate)1212 break;1213 }1214 1215 if (emc->timings[i].rate < min_rate)1216 continue;1217 1218 timing = &emc->timings[i];1219 break;1220 }1221 1222 if (!timing) {1223 dev_err(emc->dev, "no timing for rate %lu min %lu max %lu\n",1224 rate, min_rate, max_rate);1225 return -EINVAL;1226 }1227 1228 return timing->rate;1229}1230 1231static void tegra_emc_rate_requests_init(struct tegra_emc *emc)1232{1233 unsigned int i;1234 1235 for (i = 0; i < EMC_RATE_TYPE_MAX; i++) {1236 emc->requested_rate[i].min_rate = 0;1237 emc->requested_rate[i].max_rate = ULONG_MAX;1238 }1239}1240 1241static int emc_request_rate(struct tegra_emc *emc,1242 unsigned long new_min_rate,1243 unsigned long new_max_rate,1244 enum emc_rate_request_type type)1245{1246 struct emc_rate_request *req = emc->requested_rate;1247 unsigned long min_rate = 0, max_rate = ULONG_MAX;1248 unsigned int i;1249 int err;1250 1251 /* select minimum and maximum rates among the requested rates */1252 for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) {1253 if (i == type) {1254 min_rate = max(new_min_rate, min_rate);1255 max_rate = min(new_max_rate, max_rate);1256 } else {1257 min_rate = max(req->min_rate, min_rate);1258 max_rate = min(req->max_rate, max_rate);1259 }1260 }1261 1262 if (min_rate > max_rate) {1263 dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n",1264 __func__, type, min_rate, max_rate);1265 return -ERANGE;1266 }1267 1268 /*1269 * EMC rate-changes should go via OPP API because it manages voltage1270 * changes.1271 */1272 err = dev_pm_opp_set_rate(emc->dev, min_rate);1273 if (err)1274 return err;1275 1276 emc->requested_rate[type].min_rate = new_min_rate;1277 emc->requested_rate[type].max_rate = new_max_rate;1278 1279 return 0;1280}1281 1282static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate,1283 enum emc_rate_request_type type)1284{1285 struct emc_rate_request *req = &emc->requested_rate[type];1286 int ret;1287 1288 mutex_lock(&emc->rate_lock);1289 ret = emc_request_rate(emc, rate, req->max_rate, type);1290 mutex_unlock(&emc->rate_lock);1291 1292 return ret;1293}1294 1295static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate,1296 enum emc_rate_request_type type)1297{1298 struct emc_rate_request *req = &emc->requested_rate[type];1299 int ret;1300 1301 mutex_lock(&emc->rate_lock);1302 ret = emc_request_rate(emc, req->min_rate, rate, type);1303 mutex_unlock(&emc->rate_lock);1304 1305 return ret;1306}1307 1308/*1309 * debugfs interface1310 *1311 * The memory controller driver exposes some files in debugfs that can be used1312 * to control the EMC frequency. The top-level directory can be found here:1313 *1314 * /sys/kernel/debug/emc1315 *1316 * It contains the following files:1317 *1318 * - available_rates: This file contains a list of valid, space-separated1319 * EMC frequencies.1320 *1321 * - min_rate: Writing a value to this file sets the given frequency as the1322 * floor of the permitted range. If this is higher than the currently1323 * configured EMC frequency, this will cause the frequency to be1324 * increased so that it stays within the valid range.1325 *1326 * - max_rate: Similarily to the min_rate file, writing a value to this file1327 * sets the given frequency as the ceiling of the permitted range. If1328 * the value is lower than the currently configured EMC frequency, this1329 * will cause the frequency to be decreased so that it stays within the1330 * valid range.1331 */1332 1333static bool tegra_emc_validate_rate(struct tegra_emc *emc, unsigned long rate)1334{1335 unsigned int i;1336 1337 for (i = 0; i < emc->num_timings; i++)1338 if (rate == emc->timings[i].rate)1339 return true;1340 1341 return false;1342}1343 1344static int tegra_emc_debug_available_rates_show(struct seq_file *s, void *data)1345{1346 struct tegra_emc *emc = s->private;1347 const char *prefix = "";1348 unsigned int i;1349 1350 for (i = 0; i < emc->num_timings; i++) {1351 seq_printf(s, "%s%lu", prefix, emc->timings[i].rate);1352 prefix = " ";1353 }1354 1355 seq_puts(s, "\n");1356 1357 return 0;1358}1359DEFINE_SHOW_ATTRIBUTE(tegra_emc_debug_available_rates);1360 1361static int tegra_emc_debug_min_rate_get(void *data, u64 *rate)1362{1363 struct tegra_emc *emc = data;1364 1365 *rate = emc->debugfs.min_rate;1366 1367 return 0;1368}1369 1370static int tegra_emc_debug_min_rate_set(void *data, u64 rate)1371{1372 struct tegra_emc *emc = data;1373 int err;1374 1375 if (!tegra_emc_validate_rate(emc, rate))1376 return -EINVAL;1377 1378 err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG);1379 if (err < 0)1380 return err;1381 1382 emc->debugfs.min_rate = rate;1383 1384 return 0;1385}1386 1387DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_min_rate_fops,1388 tegra_emc_debug_min_rate_get,1389 tegra_emc_debug_min_rate_set, "%llu\n");1390 1391static int tegra_emc_debug_max_rate_get(void *data, u64 *rate)1392{1393 struct tegra_emc *emc = data;1394 1395 *rate = emc->debugfs.max_rate;1396 1397 return 0;1398}1399 1400static int tegra_emc_debug_max_rate_set(void *data, u64 rate)1401{1402 struct tegra_emc *emc = data;1403 int err;1404 1405 if (!tegra_emc_validate_rate(emc, rate))1406 return -EINVAL;1407 1408 err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG);1409 if (err < 0)1410 return err;1411 1412 emc->debugfs.max_rate = rate;1413 1414 return 0;1415}1416 1417DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_max_rate_fops,1418 tegra_emc_debug_max_rate_get,1419 tegra_emc_debug_max_rate_set, "%llu\n");1420 1421static void tegra_emc_debugfs_init(struct tegra_emc *emc)1422{1423 struct device *dev = emc->dev;1424 unsigned int i;1425 int err;1426 1427 emc->debugfs.min_rate = ULONG_MAX;1428 emc->debugfs.max_rate = 0;1429 1430 for (i = 0; i < emc->num_timings; i++) {1431 if (emc->timings[i].rate < emc->debugfs.min_rate)1432 emc->debugfs.min_rate = emc->timings[i].rate;1433 1434 if (emc->timings[i].rate > emc->debugfs.max_rate)1435 emc->debugfs.max_rate = emc->timings[i].rate;1436 }1437 1438 if (!emc->num_timings) {1439 emc->debugfs.min_rate = clk_get_rate(emc->clk);1440 emc->debugfs.max_rate = emc->debugfs.min_rate;1441 }1442 1443 err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,1444 emc->debugfs.max_rate);1445 if (err < 0) {1446 dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",1447 emc->debugfs.min_rate, emc->debugfs.max_rate,1448 emc->clk);1449 }1450 1451 emc->debugfs.root = debugfs_create_dir("emc", NULL);1452 1453 debugfs_create_file("available_rates", 0444, emc->debugfs.root,1454 emc, &tegra_emc_debug_available_rates_fops);1455 debugfs_create_file("min_rate", 0644, emc->debugfs.root,1456 emc, &tegra_emc_debug_min_rate_fops);1457 debugfs_create_file("max_rate", 0644, emc->debugfs.root,1458 emc, &tegra_emc_debug_max_rate_fops);1459}1460 1461static inline struct tegra_emc *1462to_tegra_emc_provider(struct icc_provider *provider)1463{1464 return container_of(provider, struct tegra_emc, provider);1465}1466 1467static struct icc_node_data *1468emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data)1469{1470 struct icc_provider *provider = data;1471 struct icc_node_data *ndata;1472 struct icc_node *node;1473 1474 /* External Memory is the only possible ICC route */1475 list_for_each_entry(node, &provider->nodes, node_list) {1476 if (node->id != TEGRA_ICC_EMEM)1477 continue;1478 1479 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);1480 if (!ndata)1481 return ERR_PTR(-ENOMEM);1482 1483 /*1484 * SRC and DST nodes should have matching TAG in order to have1485 * it set by default for a requested path.1486 */1487 ndata->tag = TEGRA_MC_ICC_TAG_ISO;1488 ndata->node = node;1489 1490 return ndata;1491 }1492 1493 return ERR_PTR(-EPROBE_DEFER);1494}1495 1496static int emc_icc_set(struct icc_node *src, struct icc_node *dst)1497{1498 struct tegra_emc *emc = to_tegra_emc_provider(dst->provider);1499 unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw);1500 unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw);1501 unsigned long long rate = max(avg_bw, peak_bw);1502 const unsigned int dram_data_bus_width_bytes = 4;1503 const unsigned int ddr = 2;1504 int err;1505 1506 /*1507 * Tegra30 EMC runs on a clock rate of SDRAM bus. This means that1508 * EMC clock rate is twice smaller than the peak data rate because1509 * data is sampled on both EMC clock edges.1510 */1511 do_div(rate, ddr * dram_data_bus_width_bytes);1512 rate = min_t(u64, rate, U32_MAX);1513 1514 err = emc_set_min_rate(emc, rate, EMC_RATE_ICC);1515 if (err)1516 return err;1517 1518 return 0;1519}1520 1521static int tegra_emc_interconnect_init(struct tegra_emc *emc)1522{1523 const struct tegra_mc_soc *soc = emc->mc->soc;1524 struct icc_node *node;1525 int err;1526 1527 emc->provider.dev = emc->dev;1528 emc->provider.set = emc_icc_set;1529 emc->provider.data = &emc->provider;1530 emc->provider.aggregate = soc->icc_ops->aggregate;1531 emc->provider.xlate_extended = emc_of_icc_xlate_extended;1532 1533 icc_provider_init(&emc->provider);1534 1535 /* create External Memory Controller node */1536 node = icc_node_create(TEGRA_ICC_EMC);1537 if (IS_ERR(node)) {1538 err = PTR_ERR(node);1539 goto err_msg;1540 }1541 1542 node->name = "External Memory Controller";1543 icc_node_add(node, &emc->provider);1544 1545 /* link External Memory Controller to External Memory (DRAM) */1546 err = icc_link_create(node, TEGRA_ICC_EMEM);1547 if (err)1548 goto remove_nodes;1549 1550 /* create External Memory node */1551 node = icc_node_create(TEGRA_ICC_EMEM);1552 if (IS_ERR(node)) {1553 err = PTR_ERR(node);1554 goto remove_nodes;1555 }1556 1557 node->name = "External Memory (DRAM)";1558 icc_node_add(node, &emc->provider);1559 1560 err = icc_provider_register(&emc->provider);1561 if (err)1562 goto remove_nodes;1563 1564 return 0;1565 1566remove_nodes:1567 icc_nodes_remove(&emc->provider);1568err_msg:1569 dev_err(emc->dev, "failed to initialize ICC: %d\n", err);1570 1571 return err;1572}1573 1574static void devm_tegra_emc_unset_callback(void *data)1575{1576 tegra20_clk_set_emc_round_callback(NULL, NULL);1577}1578 1579static void devm_tegra_emc_unreg_clk_notifier(void *data)1580{1581 struct tegra_emc *emc = data;1582 1583 clk_notifier_unregister(emc->clk, &emc->clk_nb);1584}1585 1586static int tegra_emc_init_clk(struct tegra_emc *emc)1587{1588 int err;1589 1590 tegra20_clk_set_emc_round_callback(emc_round_rate, emc);1591 1592 err = devm_add_action_or_reset(emc->dev, devm_tegra_emc_unset_callback,1593 NULL);1594 if (err)1595 return err;1596 1597 emc->clk = devm_clk_get(emc->dev, NULL);1598 if (IS_ERR(emc->clk)) {1599 dev_err(emc->dev, "failed to get EMC clock: %pe\n", emc->clk);1600 return PTR_ERR(emc->clk);1601 }1602 1603 err = clk_notifier_register(emc->clk, &emc->clk_nb);1604 if (err) {1605 dev_err(emc->dev, "failed to register clk notifier: %d\n", err);1606 return err;1607 }1608 1609 err = devm_add_action_or_reset(emc->dev,1610 devm_tegra_emc_unreg_clk_notifier, emc);1611 if (err)1612 return err;1613 1614 return 0;1615}1616 1617static int tegra_emc_probe(struct platform_device *pdev)1618{1619 struct tegra_core_opp_params opp_params = {};1620 struct device_node *np;1621 struct tegra_emc *emc;1622 int err;1623 1624 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);1625 if (!emc)1626 return -ENOMEM;1627 1628 emc->mc = devm_tegra_memory_controller_get(&pdev->dev);1629 if (IS_ERR(emc->mc))1630 return PTR_ERR(emc->mc);1631 1632 mutex_init(&emc->rate_lock);1633 emc->clk_nb.notifier_call = emc_clk_change_notify;1634 emc->dev = &pdev->dev;1635 1636 emc->regs = devm_platform_ioremap_resource(pdev, 0);1637 if (IS_ERR(emc->regs))1638 return PTR_ERR(emc->regs);1639 1640 err = emc_setup_hw(emc);1641 if (err)1642 return err;1643 1644 np = emc_find_node_by_ram_code(emc);1645 if (np) {1646 err = emc_load_timings_from_dt(emc, np);1647 of_node_put(np);1648 if (err)1649 return err;1650 }1651 1652 err = platform_get_irq(pdev, 0);1653 if (err < 0)1654 return err;1655 1656 emc->irq = err;1657 1658 err = devm_request_irq(&pdev->dev, emc->irq, tegra_emc_isr, 0,1659 dev_name(&pdev->dev), emc);1660 if (err) {1661 dev_err(&pdev->dev, "failed to request irq: %d\n", err);1662 return err;1663 }1664 1665 err = tegra_emc_init_clk(emc);1666 if (err)1667 return err;1668 1669 opp_params.init_state = true;1670 1671 err = devm_tegra_core_dev_init_opp_table(&pdev->dev, &opp_params);1672 if (err)1673 return err;1674 1675 platform_set_drvdata(pdev, emc);1676 tegra_emc_rate_requests_init(emc);1677 tegra_emc_debugfs_init(emc);1678 tegra_emc_interconnect_init(emc);1679 1680 /*1681 * Don't allow the kernel module to be unloaded. Unloading adds some1682 * extra complexity which doesn't really worth the effort in a case of1683 * this driver.1684 */1685 try_module_get(THIS_MODULE);1686 1687 return 0;1688}1689 1690static int tegra_emc_suspend(struct device *dev)1691{1692 struct tegra_emc *emc = dev_get_drvdata(dev);1693 int err;1694 1695 /* take exclusive control over the clock's rate */1696 err = clk_rate_exclusive_get(emc->clk);1697 if (err) {1698 dev_err(emc->dev, "failed to acquire clk: %d\n", err);1699 return err;1700 }1701 1702 /* suspending in a bad state will hang machine */1703 if (WARN(emc->bad_state, "hardware in a bad state\n"))1704 return -EINVAL;1705 1706 emc->bad_state = true;1707 1708 return 0;1709}1710 1711static int tegra_emc_resume(struct device *dev)1712{1713 struct tegra_emc *emc = dev_get_drvdata(dev);1714 1715 emc_setup_hw(emc);1716 emc->bad_state = false;1717 1718 clk_rate_exclusive_put(emc->clk);1719 1720 return 0;1721}1722 1723static const struct dev_pm_ops tegra_emc_pm_ops = {1724 .suspend = tegra_emc_suspend,1725 .resume = tegra_emc_resume,1726};1727 1728static const struct of_device_id tegra_emc_of_match[] = {1729 { .compatible = "nvidia,tegra30-emc", },1730 {},1731};1732MODULE_DEVICE_TABLE(of, tegra_emc_of_match);1733 1734static struct platform_driver tegra_emc_driver = {1735 .probe = tegra_emc_probe,1736 .driver = {1737 .name = "tegra30-emc",1738 .of_match_table = tegra_emc_of_match,1739 .pm = &tegra_emc_pm_ops,1740 .suppress_bind_attrs = true,1741 .sync_state = icc_sync_state,1742 },1743};1744module_platform_driver(tegra_emc_driver);1745 1746MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>");1747MODULE_DESCRIPTION("NVIDIA Tegra30 EMC driver");1748MODULE_LICENSE("GPL v2");1749