1533 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.4 *5 * Author:6 * Mikko Perttunen <mperttunen@nvidia.com>7 */8 9#include <linux/clk-provider.h>10#include <linux/clk.h>11#include <linux/clkdev.h>12#include <linux/clk/tegra.h>13#include <linux/debugfs.h>14#include <linux/delay.h>15#include <linux/interconnect-provider.h>16#include <linux/io.h>17#include <linux/module.h>18#include <linux/mutex.h>19#include <linux/of_address.h>20#include <linux/of_platform.h>21#include <linux/platform_device.h>22#include <linux/pm_opp.h>23#include <linux/sort.h>24#include <linux/string.h>25 26#include <soc/tegra/fuse.h>27#include <soc/tegra/mc.h>28 29#include "mc.h"30 31#define EMC_FBIO_CFG5 0x10432#define EMC_FBIO_CFG5_DRAM_TYPE_MASK 0x333#define EMC_FBIO_CFG5_DRAM_TYPE_SHIFT 034#define EMC_FBIO_CFG5_DRAM_WIDTH_X64 BIT(4)35 36#define EMC_INTSTATUS 0x037#define EMC_INTSTATUS_CLKCHANGE_COMPLETE BIT(4)38 39#define EMC_CFG 0xc40#define EMC_CFG_DRAM_CLKSTOP_PD BIT(31)41#define EMC_CFG_DRAM_CLKSTOP_SR BIT(30)42#define EMC_CFG_DRAM_ACPD BIT(29)43#define EMC_CFG_DYN_SREF BIT(28)44#define EMC_CFG_PWR_MASK ((0xF << 28) | BIT(18))45#define EMC_CFG_DSR_VTTGEN_DRV_EN BIT(18)46 47#define EMC_REFCTRL 0x2048#define EMC_REFCTRL_DEV_SEL_SHIFT 049#define EMC_REFCTRL_ENABLE BIT(31)50 51#define EMC_TIMING_CONTROL 0x2852#define EMC_RC 0x2c53#define EMC_RFC 0x3054#define EMC_RAS 0x3455#define EMC_RP 0x3856#define EMC_R2W 0x3c57#define EMC_W2R 0x4058#define EMC_R2P 0x4459#define EMC_W2P 0x4860#define EMC_RD_RCD 0x4c61#define EMC_WR_RCD 0x5062#define EMC_RRD 0x5463#define EMC_REXT 0x5864#define EMC_WDV 0x5c65#define EMC_QUSE 0x6066#define EMC_QRST 0x6467#define EMC_QSAFE 0x6868#define EMC_RDV 0x6c69#define EMC_REFRESH 0x7070#define EMC_BURST_REFRESH_NUM 0x7471#define EMC_PDEX2WR 0x7872#define EMC_PDEX2RD 0x7c73#define EMC_PCHG2PDEN 0x8074#define EMC_ACT2PDEN 0x8475#define EMC_AR2PDEN 0x8876#define EMC_RW2PDEN 0x8c77#define EMC_TXSR 0x9078#define EMC_TCKE 0x9479#define EMC_TFAW 0x9880#define EMC_TRPAB 0x9c81#define EMC_TCLKSTABLE 0xa082#define EMC_TCLKSTOP 0xa483#define EMC_TREFBW 0xa884#define EMC_ODT_WRITE 0xb085#define EMC_ODT_READ 0xb486#define EMC_WEXT 0xb887#define EMC_CTT 0xbc88#define EMC_RFC_SLR 0xc089#define EMC_MRS_WAIT_CNT2 0xc490 91#define EMC_MRS_WAIT_CNT 0xc892#define EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT 093#define EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK \94 (0x3FF << EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)95#define EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT 1696#define EMC_MRS_WAIT_CNT_LONG_WAIT_MASK \97 (0x3FF << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT)98 99#define EMC_MRS 0xcc100#define EMC_MODE_SET_DLL_RESET BIT(8)101#define EMC_MODE_SET_LONG_CNT BIT(26)102#define EMC_EMRS 0xd0103#define EMC_REF 0xd4104#define EMC_PRE 0xd8105 106#define EMC_SELF_REF 0xe0107#define EMC_SELF_REF_CMD_ENABLED BIT(0)108#define EMC_SELF_REF_DEV_SEL_SHIFT 30109 110#define EMC_MRW 0xe8111 112#define EMC_MRR 0xec113#define EMC_MRR_MA_SHIFT 16114#define LPDDR2_MR4_TEMP_SHIFT 0115 116#define EMC_XM2DQSPADCTRL3 0xf8117#define EMC_FBIO_SPARE 0x100118 119#define EMC_FBIO_CFG6 0x114120#define EMC_EMRS2 0x12c121#define EMC_MRW2 0x134122#define EMC_MRW4 0x13c123#define EMC_EINPUT 0x14c124#define EMC_EINPUT_DURATION 0x150125#define EMC_PUTERM_EXTRA 0x154126#define EMC_TCKESR 0x158127#define EMC_TPD 0x15c128 129#define EMC_AUTO_CAL_CONFIG 0x2a4130#define EMC_AUTO_CAL_CONFIG_AUTO_CAL_START BIT(31)131#define EMC_AUTO_CAL_INTERVAL 0x2a8132#define EMC_AUTO_CAL_STATUS 0x2ac133#define EMC_AUTO_CAL_STATUS_ACTIVE BIT(31)134#define EMC_STATUS 0x2b4135#define EMC_STATUS_TIMING_UPDATE_STALLED BIT(23)136 137#define EMC_CFG_2 0x2b8138#define EMC_CFG_2_MODE_SHIFT 0139#define EMC_CFG_2_DIS_STP_OB_CLK_DURING_NON_WR BIT(6)140 141#define EMC_CFG_DIG_DLL 0x2bc142#define EMC_CFG_DIG_DLL_PERIOD 0x2c0143#define EMC_RDV_MASK 0x2cc144#define EMC_WDV_MASK 0x2d0145#define EMC_CTT_DURATION 0x2d8146#define EMC_CTT_TERM_CTRL 0x2dc147#define EMC_ZCAL_INTERVAL 0x2e0148#define EMC_ZCAL_WAIT_CNT 0x2e4149 150#define EMC_ZQ_CAL 0x2ec151#define EMC_ZQ_CAL_CMD BIT(0)152#define EMC_ZQ_CAL_LONG BIT(4)153#define EMC_ZQ_CAL_LONG_CMD_DEV0 \154 (DRAM_DEV_SEL_0 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD)155#define EMC_ZQ_CAL_LONG_CMD_DEV1 \156 (DRAM_DEV_SEL_1 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD)157 158#define EMC_XM2CMDPADCTRL 0x2f0159#define EMC_XM2DQSPADCTRL 0x2f8160#define EMC_XM2DQSPADCTRL2 0x2fc161#define EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE BIT(0)162#define EMC_XM2DQSPADCTRL2_VREF_ENABLE BIT(5)163#define EMC_XM2DQPADCTRL 0x300164#define EMC_XM2DQPADCTRL2 0x304165#define EMC_XM2CLKPADCTRL 0x308166#define EMC_XM2COMPPADCTRL 0x30c167#define EMC_XM2VTTGENPADCTRL 0x310168#define EMC_XM2VTTGENPADCTRL2 0x314169#define EMC_XM2VTTGENPADCTRL3 0x318170#define EMC_XM2DQSPADCTRL4 0x320171#define EMC_DLL_XFORM_DQS0 0x328172#define EMC_DLL_XFORM_DQS1 0x32c173#define EMC_DLL_XFORM_DQS2 0x330174#define EMC_DLL_XFORM_DQS3 0x334175#define EMC_DLL_XFORM_DQS4 0x338176#define EMC_DLL_XFORM_DQS5 0x33c177#define EMC_DLL_XFORM_DQS6 0x340178#define EMC_DLL_XFORM_DQS7 0x344179#define EMC_DLL_XFORM_QUSE0 0x348180#define EMC_DLL_XFORM_QUSE1 0x34c181#define EMC_DLL_XFORM_QUSE2 0x350182#define EMC_DLL_XFORM_QUSE3 0x354183#define EMC_DLL_XFORM_QUSE4 0x358184#define EMC_DLL_XFORM_QUSE5 0x35c185#define EMC_DLL_XFORM_QUSE6 0x360186#define EMC_DLL_XFORM_QUSE7 0x364187#define EMC_DLL_XFORM_DQ0 0x368188#define EMC_DLL_XFORM_DQ1 0x36c189#define EMC_DLL_XFORM_DQ2 0x370190#define EMC_DLL_XFORM_DQ3 0x374191#define EMC_DLI_TRIM_TXDQS0 0x3a8192#define EMC_DLI_TRIM_TXDQS1 0x3ac193#define EMC_DLI_TRIM_TXDQS2 0x3b0194#define EMC_DLI_TRIM_TXDQS3 0x3b4195#define EMC_DLI_TRIM_TXDQS4 0x3b8196#define EMC_DLI_TRIM_TXDQS5 0x3bc197#define EMC_DLI_TRIM_TXDQS6 0x3c0198#define EMC_DLI_TRIM_TXDQS7 0x3c4199#define EMC_STALL_THEN_EXE_AFTER_CLKCHANGE 0x3cc200#define EMC_SEL_DPD_CTRL 0x3d8201#define EMC_SEL_DPD_CTRL_DATA_SEL_DPD BIT(8)202#define EMC_SEL_DPD_CTRL_ODT_SEL_DPD BIT(5)203#define EMC_SEL_DPD_CTRL_RESET_SEL_DPD BIT(4)204#define EMC_SEL_DPD_CTRL_CA_SEL_DPD BIT(3)205#define EMC_SEL_DPD_CTRL_CLK_SEL_DPD BIT(2)206#define EMC_SEL_DPD_CTRL_DDR3_MASK \207 ((0xf << 2) | BIT(8))208#define EMC_SEL_DPD_CTRL_MASK \209 ((0x3 << 2) | BIT(5) | BIT(8))210#define EMC_PRE_REFRESH_REQ_CNT 0x3dc211#define EMC_DYN_SELF_REF_CONTROL 0x3e0212#define EMC_TXSRDLL 0x3e4213#define EMC_CCFIFO_ADDR 0x3e8214#define EMC_CCFIFO_DATA 0x3ec215#define EMC_CCFIFO_STATUS 0x3f0216#define EMC_CDB_CNTL_1 0x3f4217#define EMC_CDB_CNTL_2 0x3f8218#define EMC_XM2CLKPADCTRL2 0x3fc219#define EMC_AUTO_CAL_CONFIG2 0x458220#define EMC_AUTO_CAL_CONFIG3 0x45c221#define EMC_IBDLY 0x468222#define EMC_DLL_XFORM_ADDR0 0x46c223#define EMC_DLL_XFORM_ADDR1 0x470224#define EMC_DLL_XFORM_ADDR2 0x474225#define EMC_DSR_VTTGEN_DRV 0x47c226#define EMC_TXDSRVTTGEN 0x480227#define EMC_XM2CMDPADCTRL4 0x484228#define EMC_XM2CMDPADCTRL5 0x488229#define EMC_DLL_XFORM_DQS8 0x4a0230#define EMC_DLL_XFORM_DQS9 0x4a4231#define EMC_DLL_XFORM_DQS10 0x4a8232#define EMC_DLL_XFORM_DQS11 0x4ac233#define EMC_DLL_XFORM_DQS12 0x4b0234#define EMC_DLL_XFORM_DQS13 0x4b4235#define EMC_DLL_XFORM_DQS14 0x4b8236#define EMC_DLL_XFORM_DQS15 0x4bc237#define EMC_DLL_XFORM_QUSE8 0x4c0238#define EMC_DLL_XFORM_QUSE9 0x4c4239#define EMC_DLL_XFORM_QUSE10 0x4c8240#define EMC_DLL_XFORM_QUSE11 0x4cc241#define EMC_DLL_XFORM_QUSE12 0x4d0242#define EMC_DLL_XFORM_QUSE13 0x4d4243#define EMC_DLL_XFORM_QUSE14 0x4d8244#define EMC_DLL_XFORM_QUSE15 0x4dc245#define EMC_DLL_XFORM_DQ4 0x4e0246#define EMC_DLL_XFORM_DQ5 0x4e4247#define EMC_DLL_XFORM_DQ6 0x4e8248#define EMC_DLL_XFORM_DQ7 0x4ec249#define EMC_DLI_TRIM_TXDQS8 0x520250#define EMC_DLI_TRIM_TXDQS9 0x524251#define EMC_DLI_TRIM_TXDQS10 0x528252#define EMC_DLI_TRIM_TXDQS11 0x52c253#define EMC_DLI_TRIM_TXDQS12 0x530254#define EMC_DLI_TRIM_TXDQS13 0x534255#define EMC_DLI_TRIM_TXDQS14 0x538256#define EMC_DLI_TRIM_TXDQS15 0x53c257#define EMC_CDB_CNTL_3 0x540258#define EMC_XM2DQSPADCTRL5 0x544259#define EMC_XM2DQSPADCTRL6 0x548260#define EMC_XM2DQPADCTRL3 0x54c261#define EMC_DLL_XFORM_ADDR3 0x550262#define EMC_DLL_XFORM_ADDR4 0x554263#define EMC_DLL_XFORM_ADDR5 0x558264#define EMC_CFG_PIPE 0x560265#define EMC_QPOP 0x564266#define EMC_QUSE_WIDTH 0x568267#define EMC_PUTERM_WIDTH 0x56c268#define EMC_BGBIAS_CTL0 0x570269#define EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_RX BIT(3)270#define EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_VTTGEN BIT(2)271#define EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD BIT(1)272#define EMC_PUTERM_ADJ 0x574273 274#define DRAM_DEV_SEL_ALL 0275#define DRAM_DEV_SEL_0 BIT(31)276#define DRAM_DEV_SEL_1 BIT(30)277 278#define EMC_CFG_POWER_FEATURES_MASK \279 (EMC_CFG_DYN_SREF | EMC_CFG_DRAM_ACPD | EMC_CFG_DRAM_CLKSTOP_SR | \280 EMC_CFG_DRAM_CLKSTOP_PD | EMC_CFG_DSR_VTTGEN_DRV_EN)281#define EMC_REFCTRL_DEV_SEL(n) (((n > 1) ? 0 : 2) << EMC_REFCTRL_DEV_SEL_SHIFT)282#define EMC_DRAM_DEV_SEL(n) ((n > 1) ? DRAM_DEV_SEL_ALL : DRAM_DEV_SEL_0)283 284/* Maximum amount of time in us. to wait for changes to become effective */285#define EMC_STATUS_UPDATE_TIMEOUT 1000286 287enum emc_dram_type {288 DRAM_TYPE_DDR3 = 0,289 DRAM_TYPE_DDR1 = 1,290 DRAM_TYPE_LPDDR3 = 2,291 DRAM_TYPE_DDR2 = 3292};293 294enum emc_dll_change {295 DLL_CHANGE_NONE,296 DLL_CHANGE_ON,297 DLL_CHANGE_OFF298};299 300static const unsigned long emc_burst_regs[] = {301 EMC_RC,302 EMC_RFC,303 EMC_RFC_SLR,304 EMC_RAS,305 EMC_RP,306 EMC_R2W,307 EMC_W2R,308 EMC_R2P,309 EMC_W2P,310 EMC_RD_RCD,311 EMC_WR_RCD,312 EMC_RRD,313 EMC_REXT,314 EMC_WEXT,315 EMC_WDV,316 EMC_WDV_MASK,317 EMC_QUSE,318 EMC_QUSE_WIDTH,319 EMC_IBDLY,320 EMC_EINPUT,321 EMC_EINPUT_DURATION,322 EMC_PUTERM_EXTRA,323 EMC_PUTERM_WIDTH,324 EMC_PUTERM_ADJ,325 EMC_CDB_CNTL_1,326 EMC_CDB_CNTL_2,327 EMC_CDB_CNTL_3,328 EMC_QRST,329 EMC_QSAFE,330 EMC_RDV,331 EMC_RDV_MASK,332 EMC_REFRESH,333 EMC_BURST_REFRESH_NUM,334 EMC_PRE_REFRESH_REQ_CNT,335 EMC_PDEX2WR,336 EMC_PDEX2RD,337 EMC_PCHG2PDEN,338 EMC_ACT2PDEN,339 EMC_AR2PDEN,340 EMC_RW2PDEN,341 EMC_TXSR,342 EMC_TXSRDLL,343 EMC_TCKE,344 EMC_TCKESR,345 EMC_TPD,346 EMC_TFAW,347 EMC_TRPAB,348 EMC_TCLKSTABLE,349 EMC_TCLKSTOP,350 EMC_TREFBW,351 EMC_FBIO_CFG6,352 EMC_ODT_WRITE,353 EMC_ODT_READ,354 EMC_FBIO_CFG5,355 EMC_CFG_DIG_DLL,356 EMC_CFG_DIG_DLL_PERIOD,357 EMC_DLL_XFORM_DQS0,358 EMC_DLL_XFORM_DQS1,359 EMC_DLL_XFORM_DQS2,360 EMC_DLL_XFORM_DQS3,361 EMC_DLL_XFORM_DQS4,362 EMC_DLL_XFORM_DQS5,363 EMC_DLL_XFORM_DQS6,364 EMC_DLL_XFORM_DQS7,365 EMC_DLL_XFORM_DQS8,366 EMC_DLL_XFORM_DQS9,367 EMC_DLL_XFORM_DQS10,368 EMC_DLL_XFORM_DQS11,369 EMC_DLL_XFORM_DQS12,370 EMC_DLL_XFORM_DQS13,371 EMC_DLL_XFORM_DQS14,372 EMC_DLL_XFORM_DQS15,373 EMC_DLL_XFORM_QUSE0,374 EMC_DLL_XFORM_QUSE1,375 EMC_DLL_XFORM_QUSE2,376 EMC_DLL_XFORM_QUSE3,377 EMC_DLL_XFORM_QUSE4,378 EMC_DLL_XFORM_QUSE5,379 EMC_DLL_XFORM_QUSE6,380 EMC_DLL_XFORM_QUSE7,381 EMC_DLL_XFORM_ADDR0,382 EMC_DLL_XFORM_ADDR1,383 EMC_DLL_XFORM_ADDR2,384 EMC_DLL_XFORM_ADDR3,385 EMC_DLL_XFORM_ADDR4,386 EMC_DLL_XFORM_ADDR5,387 EMC_DLL_XFORM_QUSE8,388 EMC_DLL_XFORM_QUSE9,389 EMC_DLL_XFORM_QUSE10,390 EMC_DLL_XFORM_QUSE11,391 EMC_DLL_XFORM_QUSE12,392 EMC_DLL_XFORM_QUSE13,393 EMC_DLL_XFORM_QUSE14,394 EMC_DLL_XFORM_QUSE15,395 EMC_DLI_TRIM_TXDQS0,396 EMC_DLI_TRIM_TXDQS1,397 EMC_DLI_TRIM_TXDQS2,398 EMC_DLI_TRIM_TXDQS3,399 EMC_DLI_TRIM_TXDQS4,400 EMC_DLI_TRIM_TXDQS5,401 EMC_DLI_TRIM_TXDQS6,402 EMC_DLI_TRIM_TXDQS7,403 EMC_DLI_TRIM_TXDQS8,404 EMC_DLI_TRIM_TXDQS9,405 EMC_DLI_TRIM_TXDQS10,406 EMC_DLI_TRIM_TXDQS11,407 EMC_DLI_TRIM_TXDQS12,408 EMC_DLI_TRIM_TXDQS13,409 EMC_DLI_TRIM_TXDQS14,410 EMC_DLI_TRIM_TXDQS15,411 EMC_DLL_XFORM_DQ0,412 EMC_DLL_XFORM_DQ1,413 EMC_DLL_XFORM_DQ2,414 EMC_DLL_XFORM_DQ3,415 EMC_DLL_XFORM_DQ4,416 EMC_DLL_XFORM_DQ5,417 EMC_DLL_XFORM_DQ6,418 EMC_DLL_XFORM_DQ7,419 EMC_XM2CMDPADCTRL,420 EMC_XM2CMDPADCTRL4,421 EMC_XM2CMDPADCTRL5,422 EMC_XM2DQPADCTRL2,423 EMC_XM2DQPADCTRL3,424 EMC_XM2CLKPADCTRL,425 EMC_XM2CLKPADCTRL2,426 EMC_XM2COMPPADCTRL,427 EMC_XM2VTTGENPADCTRL,428 EMC_XM2VTTGENPADCTRL2,429 EMC_XM2VTTGENPADCTRL3,430 EMC_XM2DQSPADCTRL3,431 EMC_XM2DQSPADCTRL4,432 EMC_XM2DQSPADCTRL5,433 EMC_XM2DQSPADCTRL6,434 EMC_DSR_VTTGEN_DRV,435 EMC_TXDSRVTTGEN,436 EMC_FBIO_SPARE,437 EMC_ZCAL_WAIT_CNT,438 EMC_MRS_WAIT_CNT2,439 EMC_CTT,440 EMC_CTT_DURATION,441 EMC_CFG_PIPE,442 EMC_DYN_SELF_REF_CONTROL,443 EMC_QPOP444};445 446struct emc_timing {447 unsigned long rate;448 449 u32 emc_burst_data[ARRAY_SIZE(emc_burst_regs)];450 451 u32 emc_auto_cal_config;452 u32 emc_auto_cal_config2;453 u32 emc_auto_cal_config3;454 u32 emc_auto_cal_interval;455 u32 emc_bgbias_ctl0;456 u32 emc_cfg;457 u32 emc_cfg_2;458 u32 emc_ctt_term_ctrl;459 u32 emc_mode_1;460 u32 emc_mode_2;461 u32 emc_mode_4;462 u32 emc_mode_reset;463 u32 emc_mrs_wait_cnt;464 u32 emc_sel_dpd_ctrl;465 u32 emc_xm2dqspadctrl2;466 u32 emc_zcal_cnt_long;467 u32 emc_zcal_interval;468};469 470enum emc_rate_request_type {471 EMC_RATE_DEBUG,472 EMC_RATE_ICC,473 EMC_RATE_TYPE_MAX,474};475 476struct emc_rate_request {477 unsigned long min_rate;478 unsigned long max_rate;479};480 481struct tegra_emc {482 struct device *dev;483 484 struct tegra_mc *mc;485 486 void __iomem *regs;487 488 struct clk *clk;489 490 enum emc_dram_type dram_type;491 unsigned int dram_bus_width;492 unsigned int dram_num;493 494 struct emc_timing last_timing;495 struct emc_timing *timings;496 unsigned int num_timings;497 498 struct {499 struct dentry *root;500 unsigned long min_rate;501 unsigned long max_rate;502 } debugfs;503 504 struct icc_provider provider;505 506 /*507 * There are multiple sources in the EMC driver which could request508 * a min/max clock rate, these rates are contained in this array.509 */510 struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX];511 512 /* protect shared rate-change code path */513 struct mutex rate_lock;514};515 516/* Timing change sequence functions */517 518static void emc_ccfifo_writel(struct tegra_emc *emc, u32 value,519 unsigned long offset)520{521 writel(value, emc->regs + EMC_CCFIFO_DATA);522 writel(offset, emc->regs + EMC_CCFIFO_ADDR);523}524 525static void emc_seq_update_timing(struct tegra_emc *emc)526{527 unsigned int i;528 u32 value;529 530 writel(1, emc->regs + EMC_TIMING_CONTROL);531 532 for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; ++i) {533 value = readl(emc->regs + EMC_STATUS);534 if ((value & EMC_STATUS_TIMING_UPDATE_STALLED) == 0)535 return;536 udelay(1);537 }538 539 dev_err(emc->dev, "timing update timed out\n");540}541 542static void emc_seq_disable_auto_cal(struct tegra_emc *emc)543{544 unsigned int i;545 u32 value;546 547 writel(0, emc->regs + EMC_AUTO_CAL_INTERVAL);548 549 for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; ++i) {550 value = readl(emc->regs + EMC_AUTO_CAL_STATUS);551 if ((value & EMC_AUTO_CAL_STATUS_ACTIVE) == 0)552 return;553 udelay(1);554 }555 556 dev_err(emc->dev, "auto cal disable timed out\n");557}558 559static void emc_seq_wait_clkchange(struct tegra_emc *emc)560{561 unsigned int i;562 u32 value;563 564 for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; ++i) {565 value = readl(emc->regs + EMC_INTSTATUS);566 if (value & EMC_INTSTATUS_CLKCHANGE_COMPLETE)567 return;568 udelay(1);569 }570 571 dev_err(emc->dev, "clock change timed out\n");572}573 574static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc,575 unsigned long rate)576{577 struct emc_timing *timing = NULL;578 unsigned int i;579 580 for (i = 0; i < emc->num_timings; i++) {581 if (emc->timings[i].rate == rate) {582 timing = &emc->timings[i];583 break;584 }585 }586 587 if (!timing) {588 dev_err(emc->dev, "no timing for rate %lu\n", rate);589 return NULL;590 }591 592 return timing;593}594 595static int tegra_emc_prepare_timing_change(struct tegra_emc *emc,596 unsigned long rate)597{598 struct emc_timing *timing = tegra_emc_find_timing(emc, rate);599 struct emc_timing *last = &emc->last_timing;600 enum emc_dll_change dll_change;601 unsigned int pre_wait = 0;602 u32 val, val2, mask;603 bool update = false;604 unsigned int i;605 606 if (!timing)607 return -ENOENT;608 609 if ((last->emc_mode_1 & 0x1) == (timing->emc_mode_1 & 0x1))610 dll_change = DLL_CHANGE_NONE;611 else if (timing->emc_mode_1 & 0x1)612 dll_change = DLL_CHANGE_ON;613 else614 dll_change = DLL_CHANGE_OFF;615 616 /* Clear CLKCHANGE_COMPLETE interrupts */617 writel(EMC_INTSTATUS_CLKCHANGE_COMPLETE, emc->regs + EMC_INTSTATUS);618 619 /* Disable dynamic self-refresh */620 val = readl(emc->regs + EMC_CFG);621 if (val & EMC_CFG_PWR_MASK) {622 val &= ~EMC_CFG_POWER_FEATURES_MASK;623 writel(val, emc->regs + EMC_CFG);624 625 pre_wait = 5;626 }627 628 /* Disable SEL_DPD_CTRL for clock change */629 if (emc->dram_type == DRAM_TYPE_DDR3)630 mask = EMC_SEL_DPD_CTRL_DDR3_MASK;631 else632 mask = EMC_SEL_DPD_CTRL_MASK;633 634 val = readl(emc->regs + EMC_SEL_DPD_CTRL);635 if (val & mask) {636 val &= ~mask;637 writel(val, emc->regs + EMC_SEL_DPD_CTRL);638 }639 640 /* Prepare DQ/DQS for clock change */641 val = readl(emc->regs + EMC_BGBIAS_CTL0);642 val2 = last->emc_bgbias_ctl0;643 if (!(timing->emc_bgbias_ctl0 &644 EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_RX) &&645 (val & EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_RX)) {646 val2 &= ~EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_RX;647 update = true;648 }649 650 if ((val & EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD) ||651 (val & EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_VTTGEN)) {652 update = true;653 }654 655 if (update) {656 writel(val2, emc->regs + EMC_BGBIAS_CTL0);657 if (pre_wait < 5)658 pre_wait = 5;659 }660 661 update = false;662 val = readl(emc->regs + EMC_XM2DQSPADCTRL2);663 if (timing->emc_xm2dqspadctrl2 & EMC_XM2DQSPADCTRL2_VREF_ENABLE &&664 !(val & EMC_XM2DQSPADCTRL2_VREF_ENABLE)) {665 val |= EMC_XM2DQSPADCTRL2_VREF_ENABLE;666 update = true;667 }668 669 if (timing->emc_xm2dqspadctrl2 & EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE &&670 !(val & EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE)) {671 val |= EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE;672 update = true;673 }674 675 if (update) {676 writel(val, emc->regs + EMC_XM2DQSPADCTRL2);677 if (pre_wait < 30)678 pre_wait = 30;679 }680 681 /* Wait to settle */682 if (pre_wait) {683 emc_seq_update_timing(emc);684 udelay(pre_wait);685 }686 687 /* Program CTT_TERM control */688 if (last->emc_ctt_term_ctrl != timing->emc_ctt_term_ctrl) {689 emc_seq_disable_auto_cal(emc);690 writel(timing->emc_ctt_term_ctrl,691 emc->regs + EMC_CTT_TERM_CTRL);692 emc_seq_update_timing(emc);693 }694 695 /* Program burst shadow registers */696 for (i = 0; i < ARRAY_SIZE(timing->emc_burst_data); ++i)697 writel(timing->emc_burst_data[i],698 emc->regs + emc_burst_regs[i]);699 700 writel(timing->emc_xm2dqspadctrl2, emc->regs + EMC_XM2DQSPADCTRL2);701 writel(timing->emc_zcal_interval, emc->regs + EMC_ZCAL_INTERVAL);702 703 tegra_mc_write_emem_configuration(emc->mc, timing->rate);704 705 val = timing->emc_cfg & ~EMC_CFG_POWER_FEATURES_MASK;706 emc_ccfifo_writel(emc, val, EMC_CFG);707 708 /* Program AUTO_CAL_CONFIG */709 if (timing->emc_auto_cal_config2 != last->emc_auto_cal_config2)710 emc_ccfifo_writel(emc, timing->emc_auto_cal_config2,711 EMC_AUTO_CAL_CONFIG2);712 713 if (timing->emc_auto_cal_config3 != last->emc_auto_cal_config3)714 emc_ccfifo_writel(emc, timing->emc_auto_cal_config3,715 EMC_AUTO_CAL_CONFIG3);716 717 if (timing->emc_auto_cal_config != last->emc_auto_cal_config) {718 val = timing->emc_auto_cal_config;719 val &= EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;720 emc_ccfifo_writel(emc, val, EMC_AUTO_CAL_CONFIG);721 }722 723 /* DDR3: predict MRS long wait count */724 if (emc->dram_type == DRAM_TYPE_DDR3 &&725 dll_change == DLL_CHANGE_ON) {726 u32 cnt = 512;727 728 if (timing->emc_zcal_interval != 0 &&729 last->emc_zcal_interval == 0)730 cnt -= emc->dram_num * 256;731 732 val = (timing->emc_mrs_wait_cnt733 & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK)734 >> EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT;735 if (cnt < val)736 cnt = val;737 738 val = timing->emc_mrs_wait_cnt739 & ~EMC_MRS_WAIT_CNT_LONG_WAIT_MASK;740 val |= (cnt << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT)741 & EMC_MRS_WAIT_CNT_LONG_WAIT_MASK;742 743 writel(val, emc->regs + EMC_MRS_WAIT_CNT);744 }745 746 val = timing->emc_cfg_2;747 val &= ~EMC_CFG_2_DIS_STP_OB_CLK_DURING_NON_WR;748 emc_ccfifo_writel(emc, val, EMC_CFG_2);749 750 /* DDR3: Turn off DLL and enter self-refresh */751 if (emc->dram_type == DRAM_TYPE_DDR3 && dll_change == DLL_CHANGE_OFF)752 emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);753 754 /* Disable refresh controller */755 emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num),756 EMC_REFCTRL);757 if (emc->dram_type == DRAM_TYPE_DDR3)758 emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num) |759 EMC_SELF_REF_CMD_ENABLED,760 EMC_SELF_REF);761 762 /* Flow control marker */763 emc_ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE);764 765 /* DDR3: Exit self-refresh */766 if (emc->dram_type == DRAM_TYPE_DDR3)767 emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num),768 EMC_SELF_REF);769 emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num) |770 EMC_REFCTRL_ENABLE,771 EMC_REFCTRL);772 773 /* Set DRAM mode registers */774 if (emc->dram_type == DRAM_TYPE_DDR3) {775 if (timing->emc_mode_1 != last->emc_mode_1)776 emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);777 if (timing->emc_mode_2 != last->emc_mode_2)778 emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_EMRS2);779 780 if ((timing->emc_mode_reset != last->emc_mode_reset) ||781 dll_change == DLL_CHANGE_ON) {782 val = timing->emc_mode_reset;783 if (dll_change == DLL_CHANGE_ON) {784 val |= EMC_MODE_SET_DLL_RESET;785 val |= EMC_MODE_SET_LONG_CNT;786 } else {787 val &= ~EMC_MODE_SET_DLL_RESET;788 }789 emc_ccfifo_writel(emc, val, EMC_MRS);790 }791 } else {792 if (timing->emc_mode_2 != last->emc_mode_2)793 emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_MRW2);794 if (timing->emc_mode_1 != last->emc_mode_1)795 emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_MRW);796 if (timing->emc_mode_4 != last->emc_mode_4)797 emc_ccfifo_writel(emc, timing->emc_mode_4, EMC_MRW4);798 }799 800 /* Issue ZCAL command if turning ZCAL on */801 if (timing->emc_zcal_interval != 0 && last->emc_zcal_interval == 0) {802 emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV0, EMC_ZQ_CAL);803 if (emc->dram_num > 1)804 emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV1,805 EMC_ZQ_CAL);806 }807 808 /* Write to RO register to remove stall after change */809 emc_ccfifo_writel(emc, 0, EMC_CCFIFO_STATUS);810 811 if (timing->emc_cfg_2 & EMC_CFG_2_DIS_STP_OB_CLK_DURING_NON_WR)812 emc_ccfifo_writel(emc, timing->emc_cfg_2, EMC_CFG_2);813 814 /* Disable AUTO_CAL for clock change */815 emc_seq_disable_auto_cal(emc);816 817 /* Read register to wait until programming has settled */818 readl(emc->regs + EMC_INTSTATUS);819 820 return 0;821}822 823static void tegra_emc_complete_timing_change(struct tegra_emc *emc,824 unsigned long rate)825{826 struct emc_timing *timing = tegra_emc_find_timing(emc, rate);827 struct emc_timing *last = &emc->last_timing;828 u32 val;829 830 if (!timing)831 return;832 833 /* Wait until the state machine has settled */834 emc_seq_wait_clkchange(emc);835 836 /* Restore AUTO_CAL */837 if (timing->emc_ctt_term_ctrl != last->emc_ctt_term_ctrl)838 writel(timing->emc_auto_cal_interval,839 emc->regs + EMC_AUTO_CAL_INTERVAL);840 841 /* Restore dynamic self-refresh */842 if (timing->emc_cfg & EMC_CFG_PWR_MASK)843 writel(timing->emc_cfg, emc->regs + EMC_CFG);844 845 /* Set ZCAL wait count */846 writel(timing->emc_zcal_cnt_long, emc->regs + EMC_ZCAL_WAIT_CNT);847 848 /* LPDDR3: Turn off BGBIAS if low frequency */849 if (emc->dram_type == DRAM_TYPE_LPDDR3 &&850 timing->emc_bgbias_ctl0 &851 EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_RX) {852 val = timing->emc_bgbias_ctl0;853 val |= EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD_IBIAS_VTTGEN;854 val |= EMC_BGBIAS_CTL0_BIAS0_DSC_E_PWRD;855 writel(val, emc->regs + EMC_BGBIAS_CTL0);856 } else {857 if (emc->dram_type == DRAM_TYPE_DDR3 &&858 readl(emc->regs + EMC_BGBIAS_CTL0) !=859 timing->emc_bgbias_ctl0) {860 writel(timing->emc_bgbias_ctl0,861 emc->regs + EMC_BGBIAS_CTL0);862 }863 864 writel(timing->emc_auto_cal_interval,865 emc->regs + EMC_AUTO_CAL_INTERVAL);866 }867 868 /* Wait for timing to settle */869 udelay(2);870 871 /* Reprogram SEL_DPD_CTRL */872 writel(timing->emc_sel_dpd_ctrl, emc->regs + EMC_SEL_DPD_CTRL);873 emc_seq_update_timing(emc);874 875 emc->last_timing = *timing;876}877 878/* Initialization and deinitialization */879 880static void emc_read_current_timing(struct tegra_emc *emc,881 struct emc_timing *timing)882{883 unsigned int i;884 885 for (i = 0; i < ARRAY_SIZE(emc_burst_regs); ++i)886 timing->emc_burst_data[i] =887 readl(emc->regs + emc_burst_regs[i]);888 889 timing->emc_cfg = readl(emc->regs + EMC_CFG);890 891 timing->emc_auto_cal_interval = 0;892 timing->emc_zcal_cnt_long = 0;893 timing->emc_mode_1 = 0;894 timing->emc_mode_2 = 0;895 timing->emc_mode_4 = 0;896 timing->emc_mode_reset = 0;897}898 899static int emc_init(struct tegra_emc *emc)900{901 emc->dram_type = readl(emc->regs + EMC_FBIO_CFG5);902 903 if (emc->dram_type & EMC_FBIO_CFG5_DRAM_WIDTH_X64)904 emc->dram_bus_width = 64;905 else906 emc->dram_bus_width = 32;907 908 dev_info_once(emc->dev, "%ubit DRAM bus\n", emc->dram_bus_width);909 910 emc->dram_type &= EMC_FBIO_CFG5_DRAM_TYPE_MASK;911 emc->dram_type >>= EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;912 913 emc->dram_num = tegra_mc_get_emem_device_count(emc->mc);914 915 emc_read_current_timing(emc, &emc->last_timing);916 917 return 0;918}919 920static int load_one_timing_from_dt(struct tegra_emc *emc,921 struct emc_timing *timing,922 struct device_node *node)923{924 u32 value;925 int err;926 927 err = of_property_read_u32(node, "clock-frequency", &value);928 if (err) {929 dev_err(emc->dev, "timing %pOFn: failed to read rate: %d\n",930 node, err);931 return err;932 }933 934 timing->rate = value;935 936 err = of_property_read_u32_array(node, "nvidia,emc-configuration",937 timing->emc_burst_data,938 ARRAY_SIZE(timing->emc_burst_data));939 if (err) {940 dev_err(emc->dev,941 "timing %pOFn: failed to read emc burst data: %d\n",942 node, err);943 return err;944 }945 946#define EMC_READ_PROP(prop, dtprop) { \947 err = of_property_read_u32(node, dtprop, &timing->prop); \948 if (err) { \949 dev_err(emc->dev, "timing %pOFn: failed to read " #prop ": %d\n", \950 node, err); \951 return err; \952 } \953}954 955 EMC_READ_PROP(emc_auto_cal_config, "nvidia,emc-auto-cal-config")956 EMC_READ_PROP(emc_auto_cal_config2, "nvidia,emc-auto-cal-config2")957 EMC_READ_PROP(emc_auto_cal_config3, "nvidia,emc-auto-cal-config3")958 EMC_READ_PROP(emc_auto_cal_interval, "nvidia,emc-auto-cal-interval")959 EMC_READ_PROP(emc_bgbias_ctl0, "nvidia,emc-bgbias-ctl0")960 EMC_READ_PROP(emc_cfg, "nvidia,emc-cfg")961 EMC_READ_PROP(emc_cfg_2, "nvidia,emc-cfg-2")962 EMC_READ_PROP(emc_ctt_term_ctrl, "nvidia,emc-ctt-term-ctrl")963 EMC_READ_PROP(emc_mode_1, "nvidia,emc-mode-1")964 EMC_READ_PROP(emc_mode_2, "nvidia,emc-mode-2")965 EMC_READ_PROP(emc_mode_4, "nvidia,emc-mode-4")966 EMC_READ_PROP(emc_mode_reset, "nvidia,emc-mode-reset")967 EMC_READ_PROP(emc_mrs_wait_cnt, "nvidia,emc-mrs-wait-cnt")968 EMC_READ_PROP(emc_sel_dpd_ctrl, "nvidia,emc-sel-dpd-ctrl")969 EMC_READ_PROP(emc_xm2dqspadctrl2, "nvidia,emc-xm2dqspadctrl2")970 EMC_READ_PROP(emc_zcal_cnt_long, "nvidia,emc-zcal-cnt-long")971 EMC_READ_PROP(emc_zcal_interval, "nvidia,emc-zcal-interval")972 973#undef EMC_READ_PROP974 975 return 0;976}977 978static int cmp_timings(const void *_a, const void *_b)979{980 const struct emc_timing *a = _a;981 const struct emc_timing *b = _b;982 983 if (a->rate < b->rate)984 return -1;985 else if (a->rate == b->rate)986 return 0;987 else988 return 1;989}990 991static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc,992 struct device_node *node)993{994 int child_count = of_get_child_count(node);995 struct emc_timing *timing;996 unsigned int i = 0;997 int err;998 999 emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing),1000 GFP_KERNEL);1001 if (!emc->timings)1002 return -ENOMEM;1003 1004 emc->num_timings = child_count;1005 1006 for_each_child_of_node_scoped(node, child) {1007 timing = &emc->timings[i++];1008 1009 err = load_one_timing_from_dt(emc, timing, child);1010 if (err)1011 return err;1012 }1013 1014 sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings,1015 NULL);1016 1017 return 0;1018}1019 1020static const struct of_device_id tegra_emc_of_match[] = {1021 { .compatible = "nvidia,tegra124-emc" },1022 { .compatible = "nvidia,tegra132-emc" },1023 {}1024};1025MODULE_DEVICE_TABLE(of, tegra_emc_of_match);1026 1027static struct device_node *1028tegra_emc_find_node_by_ram_code(struct device_node *node, u32 ram_code)1029{1030 struct device_node *np;1031 int err;1032 1033 for_each_child_of_node(node, np) {1034 u32 value;1035 1036 err = of_property_read_u32(np, "nvidia,ram-code", &value);1037 if (err || (value != ram_code))1038 continue;1039 1040 return np;1041 }1042 1043 return NULL;1044}1045 1046static void tegra_emc_rate_requests_init(struct tegra_emc *emc)1047{1048 unsigned int i;1049 1050 for (i = 0; i < EMC_RATE_TYPE_MAX; i++) {1051 emc->requested_rate[i].min_rate = 0;1052 emc->requested_rate[i].max_rate = ULONG_MAX;1053 }1054}1055 1056static int emc_request_rate(struct tegra_emc *emc,1057 unsigned long new_min_rate,1058 unsigned long new_max_rate,1059 enum emc_rate_request_type type)1060{1061 struct emc_rate_request *req = emc->requested_rate;1062 unsigned long min_rate = 0, max_rate = ULONG_MAX;1063 unsigned int i;1064 int err;1065 1066 /* select minimum and maximum rates among the requested rates */1067 for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) {1068 if (i == type) {1069 min_rate = max(new_min_rate, min_rate);1070 max_rate = min(new_max_rate, max_rate);1071 } else {1072 min_rate = max(req->min_rate, min_rate);1073 max_rate = min(req->max_rate, max_rate);1074 }1075 }1076 1077 if (min_rate > max_rate) {1078 dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n",1079 __func__, type, min_rate, max_rate);1080 return -ERANGE;1081 }1082 1083 /*1084 * EMC rate-changes should go via OPP API because it manages voltage1085 * changes.1086 */1087 err = dev_pm_opp_set_rate(emc->dev, min_rate);1088 if (err)1089 return err;1090 1091 emc->requested_rate[type].min_rate = new_min_rate;1092 emc->requested_rate[type].max_rate = new_max_rate;1093 1094 return 0;1095}1096 1097static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate,1098 enum emc_rate_request_type type)1099{1100 struct emc_rate_request *req = &emc->requested_rate[type];1101 int ret;1102 1103 mutex_lock(&emc->rate_lock);1104 ret = emc_request_rate(emc, rate, req->max_rate, type);1105 mutex_unlock(&emc->rate_lock);1106 1107 return ret;1108}1109 1110static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate,1111 enum emc_rate_request_type type)1112{1113 struct emc_rate_request *req = &emc->requested_rate[type];1114 int ret;1115 1116 mutex_lock(&emc->rate_lock);1117 ret = emc_request_rate(emc, req->min_rate, rate, type);1118 mutex_unlock(&emc->rate_lock);1119 1120 return ret;1121}1122 1123/*1124 * debugfs interface1125 *1126 * The memory controller driver exposes some files in debugfs that can be used1127 * to control the EMC frequency. The top-level directory can be found here:1128 *1129 * /sys/kernel/debug/emc1130 *1131 * It contains the following files:1132 *1133 * - available_rates: This file contains a list of valid, space-separated1134 * EMC frequencies.1135 *1136 * - min_rate: Writing a value to this file sets the given frequency as the1137 * floor of the permitted range. If this is higher than the currently1138 * configured EMC frequency, this will cause the frequency to be1139 * increased so that it stays within the valid range.1140 *1141 * - max_rate: Similarily to the min_rate file, writing a value to this file1142 * sets the given frequency as the ceiling of the permitted range. If1143 * the value is lower than the currently configured EMC frequency, this1144 * will cause the frequency to be decreased so that it stays within the1145 * valid range.1146 */1147 1148static bool tegra_emc_validate_rate(struct tegra_emc *emc, unsigned long rate)1149{1150 unsigned int i;1151 1152 for (i = 0; i < emc->num_timings; i++)1153 if (rate == emc->timings[i].rate)1154 return true;1155 1156 return false;1157}1158 1159static int tegra_emc_debug_available_rates_show(struct seq_file *s,1160 void *data)1161{1162 struct tegra_emc *emc = s->private;1163 const char *prefix = "";1164 unsigned int i;1165 1166 for (i = 0; i < emc->num_timings; i++) {1167 seq_printf(s, "%s%lu", prefix, emc->timings[i].rate);1168 prefix = " ";1169 }1170 1171 seq_puts(s, "\n");1172 1173 return 0;1174}1175 1176DEFINE_SHOW_ATTRIBUTE(tegra_emc_debug_available_rates);1177 1178static int tegra_emc_debug_min_rate_get(void *data, u64 *rate)1179{1180 struct tegra_emc *emc = data;1181 1182 *rate = emc->debugfs.min_rate;1183 1184 return 0;1185}1186 1187static int tegra_emc_debug_min_rate_set(void *data, u64 rate)1188{1189 struct tegra_emc *emc = data;1190 int err;1191 1192 if (!tegra_emc_validate_rate(emc, rate))1193 return -EINVAL;1194 1195 err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG);1196 if (err < 0)1197 return err;1198 1199 emc->debugfs.min_rate = rate;1200 1201 return 0;1202}1203 1204DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_min_rate_fops,1205 tegra_emc_debug_min_rate_get,1206 tegra_emc_debug_min_rate_set, "%llu\n");1207 1208static int tegra_emc_debug_max_rate_get(void *data, u64 *rate)1209{1210 struct tegra_emc *emc = data;1211 1212 *rate = emc->debugfs.max_rate;1213 1214 return 0;1215}1216 1217static int tegra_emc_debug_max_rate_set(void *data, u64 rate)1218{1219 struct tegra_emc *emc = data;1220 int err;1221 1222 if (!tegra_emc_validate_rate(emc, rate))1223 return -EINVAL;1224 1225 err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG);1226 if (err < 0)1227 return err;1228 1229 emc->debugfs.max_rate = rate;1230 1231 return 0;1232}1233 1234DEFINE_DEBUGFS_ATTRIBUTE(tegra_emc_debug_max_rate_fops,1235 tegra_emc_debug_max_rate_get,1236 tegra_emc_debug_max_rate_set, "%llu\n");1237 1238static void emc_debugfs_init(struct device *dev, struct tegra_emc *emc)1239{1240 unsigned int i;1241 int err;1242 1243 emc->debugfs.min_rate = ULONG_MAX;1244 emc->debugfs.max_rate = 0;1245 1246 for (i = 0; i < emc->num_timings; i++) {1247 if (emc->timings[i].rate < emc->debugfs.min_rate)1248 emc->debugfs.min_rate = emc->timings[i].rate;1249 1250 if (emc->timings[i].rate > emc->debugfs.max_rate)1251 emc->debugfs.max_rate = emc->timings[i].rate;1252 }1253 1254 if (!emc->num_timings) {1255 emc->debugfs.min_rate = clk_get_rate(emc->clk);1256 emc->debugfs.max_rate = emc->debugfs.min_rate;1257 }1258 1259 err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,1260 emc->debugfs.max_rate);1261 if (err < 0) {1262 dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",1263 emc->debugfs.min_rate, emc->debugfs.max_rate,1264 emc->clk);1265 return;1266 }1267 1268 emc->debugfs.root = debugfs_create_dir("emc", NULL);1269 1270 debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,1271 &tegra_emc_debug_available_rates_fops);1272 debugfs_create_file("min_rate", 0644, emc->debugfs.root,1273 emc, &tegra_emc_debug_min_rate_fops);1274 debugfs_create_file("max_rate", 0644, emc->debugfs.root,1275 emc, &tegra_emc_debug_max_rate_fops);1276}1277 1278static inline struct tegra_emc *1279to_tegra_emc_provider(struct icc_provider *provider)1280{1281 return container_of(provider, struct tegra_emc, provider);1282}1283 1284static struct icc_node_data *1285emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data)1286{1287 struct icc_provider *provider = data;1288 struct icc_node_data *ndata;1289 struct icc_node *node;1290 1291 /* External Memory is the only possible ICC route */1292 list_for_each_entry(node, &provider->nodes, node_list) {1293 if (node->id != TEGRA_ICC_EMEM)1294 continue;1295 1296 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);1297 if (!ndata)1298 return ERR_PTR(-ENOMEM);1299 1300 /*1301 * SRC and DST nodes should have matching TAG in order to have1302 * it set by default for a requested path.1303 */1304 ndata->tag = TEGRA_MC_ICC_TAG_ISO;1305 ndata->node = node;1306 1307 return ndata;1308 }1309 1310 return ERR_PTR(-EPROBE_DEFER);1311}1312 1313static int emc_icc_set(struct icc_node *src, struct icc_node *dst)1314{1315 struct tegra_emc *emc = to_tegra_emc_provider(dst->provider);1316 unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw);1317 unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw);1318 unsigned long long rate = max(avg_bw, peak_bw);1319 unsigned int dram_data_bus_width_bytes;1320 const unsigned int ddr = 2;1321 int err;1322 1323 /*1324 * Tegra124 EMC runs on a clock rate of SDRAM bus. This means that1325 * EMC clock rate is twice smaller than the peak data rate because1326 * data is sampled on both EMC clock edges.1327 */1328 dram_data_bus_width_bytes = emc->dram_bus_width / 8;1329 do_div(rate, ddr * dram_data_bus_width_bytes);1330 rate = min_t(u64, rate, U32_MAX);1331 1332 err = emc_set_min_rate(emc, rate, EMC_RATE_ICC);1333 if (err)1334 return err;1335 1336 return 0;1337}1338 1339static int tegra_emc_interconnect_init(struct tegra_emc *emc)1340{1341 const struct tegra_mc_soc *soc = emc->mc->soc;1342 struct icc_node *node;1343 int err;1344 1345 emc->provider.dev = emc->dev;1346 emc->provider.set = emc_icc_set;1347 emc->provider.data = &emc->provider;1348 emc->provider.aggregate = soc->icc_ops->aggregate;1349 emc->provider.xlate_extended = emc_of_icc_xlate_extended;1350 1351 icc_provider_init(&emc->provider);1352 1353 /* create External Memory Controller node */1354 node = icc_node_create(TEGRA_ICC_EMC);1355 if (IS_ERR(node)) {1356 err = PTR_ERR(node);1357 goto err_msg;1358 }1359 1360 node->name = "External Memory Controller";1361 icc_node_add(node, &emc->provider);1362 1363 /* link External Memory Controller to External Memory (DRAM) */1364 err = icc_link_create(node, TEGRA_ICC_EMEM);1365 if (err)1366 goto remove_nodes;1367 1368 /* create External Memory node */1369 node = icc_node_create(TEGRA_ICC_EMEM);1370 if (IS_ERR(node)) {1371 err = PTR_ERR(node);1372 goto remove_nodes;1373 }1374 1375 node->name = "External Memory (DRAM)";1376 icc_node_add(node, &emc->provider);1377 1378 err = icc_provider_register(&emc->provider);1379 if (err)1380 goto remove_nodes;1381 1382 return 0;1383 1384remove_nodes:1385 icc_nodes_remove(&emc->provider);1386err_msg:1387 dev_err(emc->dev, "failed to initialize ICC: %d\n", err);1388 1389 return err;1390}1391 1392static int tegra_emc_opp_table_init(struct tegra_emc *emc)1393{1394 u32 hw_version = BIT(tegra_sku_info.soc_speedo_id);1395 int opp_token, err;1396 1397 err = dev_pm_opp_set_supported_hw(emc->dev, &hw_version, 1);1398 if (err < 0) {1399 dev_err(emc->dev, "failed to set OPP supported HW: %d\n", err);1400 return err;1401 }1402 opp_token = err;1403 1404 err = dev_pm_opp_of_add_table(emc->dev);1405 if (err) {1406 if (err == -ENODEV)1407 dev_err(emc->dev, "OPP table not found, please update your device tree\n");1408 else1409 dev_err(emc->dev, "failed to add OPP table: %d\n", err);1410 1411 goto put_hw_table;1412 }1413 1414 dev_info_once(emc->dev, "OPP HW ver. 0x%x, current clock rate %lu MHz\n",1415 hw_version, clk_get_rate(emc->clk) / 1000000);1416 1417 /* first dummy rate-set initializes voltage state */1418 err = dev_pm_opp_set_rate(emc->dev, clk_get_rate(emc->clk));1419 if (err) {1420 dev_err(emc->dev, "failed to initialize OPP clock: %d\n", err);1421 goto remove_table;1422 }1423 1424 return 0;1425 1426remove_table:1427 dev_pm_opp_of_remove_table(emc->dev);1428put_hw_table:1429 dev_pm_opp_put_supported_hw(opp_token);1430 1431 return err;1432}1433 1434static void devm_tegra_emc_unset_callback(void *data)1435{1436 tegra124_clk_set_emc_callbacks(NULL, NULL);1437}1438 1439static int tegra_emc_probe(struct platform_device *pdev)1440{1441 struct device_node *np;1442 struct tegra_emc *emc;1443 u32 ram_code;1444 int err;1445 1446 emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);1447 if (!emc)1448 return -ENOMEM;1449 1450 mutex_init(&emc->rate_lock);1451 emc->dev = &pdev->dev;1452 1453 emc->regs = devm_platform_ioremap_resource(pdev, 0);1454 if (IS_ERR(emc->regs))1455 return PTR_ERR(emc->regs);1456 1457 emc->mc = devm_tegra_memory_controller_get(&pdev->dev);1458 if (IS_ERR(emc->mc))1459 return PTR_ERR(emc->mc);1460 1461 ram_code = tegra_read_ram_code();1462 1463 np = tegra_emc_find_node_by_ram_code(pdev->dev.of_node, ram_code);1464 if (np) {1465 err = tegra_emc_load_timings_from_dt(emc, np);1466 of_node_put(np);1467 if (err)1468 return err;1469 } else {1470 dev_info_once(&pdev->dev,1471 "no memory timings for RAM code %u found in DT\n",1472 ram_code);1473 }1474 1475 err = emc_init(emc);1476 if (err) {1477 dev_err(&pdev->dev, "EMC initialization failed: %d\n", err);1478 return err;1479 }1480 1481 platform_set_drvdata(pdev, emc);1482 1483 tegra124_clk_set_emc_callbacks(tegra_emc_prepare_timing_change,1484 tegra_emc_complete_timing_change);1485 1486 err = devm_add_action_or_reset(&pdev->dev, devm_tegra_emc_unset_callback,1487 NULL);1488 if (err)1489 return err;1490 1491 emc->clk = devm_clk_get(&pdev->dev, "emc");1492 if (IS_ERR(emc->clk)) {1493 err = PTR_ERR(emc->clk);1494 dev_err(&pdev->dev, "failed to get EMC clock: %d\n", err);1495 return err;1496 }1497 1498 err = tegra_emc_opp_table_init(emc);1499 if (err)1500 return err;1501 1502 tegra_emc_rate_requests_init(emc);1503 1504 if (IS_ENABLED(CONFIG_DEBUG_FS))1505 emc_debugfs_init(&pdev->dev, emc);1506 1507 tegra_emc_interconnect_init(emc);1508 1509 /*1510 * Don't allow the kernel module to be unloaded. Unloading adds some1511 * extra complexity which doesn't really worth the effort in a case of1512 * this driver.1513 */1514 try_module_get(THIS_MODULE);1515 1516 return 0;1517};1518 1519static struct platform_driver tegra_emc_driver = {1520 .probe = tegra_emc_probe,1521 .driver = {1522 .name = "tegra-emc",1523 .of_match_table = tegra_emc_of_match,1524 .suppress_bind_attrs = true,1525 .sync_state = icc_sync_state,1526 },1527};1528module_platform_driver(tegra_emc_driver);1529 1530MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>");1531MODULE_DESCRIPTION("NVIDIA Tegra124 EMC driver");1532MODULE_LICENSE("GPL v2");1533