1176 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * EMIF driver4 *5 * Copyright (C) 2012 Texas Instruments, Inc.6 *7 * Aneesh V <aneesh@ti.com>8 * Santosh Shilimkar <santosh.shilimkar@ti.com>9 */10#include <linux/cleanup.h>11#include <linux/err.h>12#include <linux/kernel.h>13#include <linux/reboot.h>14#include <linux/platform_data/emif_plat.h>15#include <linux/io.h>16#include <linux/device.h>17#include <linux/platform_device.h>18#include <linux/interrupt.h>19#include <linux/slab.h>20#include <linux/of.h>21#include <linux/debugfs.h>22#include <linux/seq_file.h>23#include <linux/module.h>24#include <linux/list.h>25#include <linux/spinlock.h>26#include <linux/pm.h>27 28#include "emif.h"29#include "jedec_ddr.h"30#include "of_memory.h"31 32/**33 * struct emif_data - Per device static data for driver's use34 * @duplicate: Whether the DDR devices attached to this EMIF35 * instance are exactly same as that on EMIF1. In36 * this case we can save some memory and processing37 * @temperature_level: Maximum temperature of LPDDR2 devices attached38 * to this EMIF - read from MR4 register. If there39 * are two devices attached to this EMIF, this40 * value is the maximum of the two temperature41 * levels.42 * @node: node in the device list43 * @base: base address of memory-mapped IO registers.44 * @dev: device pointer.45 * @regs_cache: An array of 'struct emif_regs' that stores46 * calculated register values for different47 * frequencies, to avoid re-calculating them on48 * each DVFS transition.49 * @curr_regs: The set of register values used in the last50 * frequency change (i.e. corresponding to the51 * frequency in effect at the moment)52 * @plat_data: Pointer to saved platform data.53 * @debugfs_root: dentry to the root folder for EMIF in debugfs54 * @np_ddr: Pointer to ddr device tree node55 */56struct emif_data {57 u8 duplicate;58 u8 temperature_level;59 u8 lpmode;60 struct list_head node;61 void __iomem *base;62 struct device *dev;63 struct emif_regs *regs_cache[EMIF_MAX_NUM_FREQUENCIES];64 struct emif_regs *curr_regs;65 struct emif_platform_data *plat_data;66 struct dentry *debugfs_root;67 struct device_node *np_ddr;68};69 70static struct emif_data *emif1;71static DEFINE_SPINLOCK(emif_lock);72static LIST_HEAD(device_list);73 74static void do_emif_regdump_show(struct seq_file *s, struct emif_data *emif,75 struct emif_regs *regs)76{77 u32 type = emif->plat_data->device_info->type;78 u32 ip_rev = emif->plat_data->ip_rev;79 80 seq_printf(s, "EMIF register cache dump for %dMHz\n",81 regs->freq/1000000);82 83 seq_printf(s, "ref_ctrl_shdw\t: 0x%08x\n", regs->ref_ctrl_shdw);84 seq_printf(s, "sdram_tim1_shdw\t: 0x%08x\n", regs->sdram_tim1_shdw);85 seq_printf(s, "sdram_tim2_shdw\t: 0x%08x\n", regs->sdram_tim2_shdw);86 seq_printf(s, "sdram_tim3_shdw\t: 0x%08x\n", regs->sdram_tim3_shdw);87 88 if (ip_rev == EMIF_4D) {89 seq_printf(s, "read_idle_ctrl_shdw_normal\t: 0x%08x\n",90 regs->read_idle_ctrl_shdw_normal);91 seq_printf(s, "read_idle_ctrl_shdw_volt_ramp\t: 0x%08x\n",92 regs->read_idle_ctrl_shdw_volt_ramp);93 } else if (ip_rev == EMIF_4D5) {94 seq_printf(s, "dll_calib_ctrl_shdw_normal\t: 0x%08x\n",95 regs->dll_calib_ctrl_shdw_normal);96 seq_printf(s, "dll_calib_ctrl_shdw_volt_ramp\t: 0x%08x\n",97 regs->dll_calib_ctrl_shdw_volt_ramp);98 }99 100 if (type == DDR_TYPE_LPDDR2_S2 || type == DDR_TYPE_LPDDR2_S4) {101 seq_printf(s, "ref_ctrl_shdw_derated\t: 0x%08x\n",102 regs->ref_ctrl_shdw_derated);103 seq_printf(s, "sdram_tim1_shdw_derated\t: 0x%08x\n",104 regs->sdram_tim1_shdw_derated);105 seq_printf(s, "sdram_tim3_shdw_derated\t: 0x%08x\n",106 regs->sdram_tim3_shdw_derated);107 }108}109 110static int emif_regdump_show(struct seq_file *s, void *unused)111{112 struct emif_data *emif = s->private;113 struct emif_regs **regs_cache;114 int i;115 116 if (emif->duplicate)117 regs_cache = emif1->regs_cache;118 else119 regs_cache = emif->regs_cache;120 121 for (i = 0; i < EMIF_MAX_NUM_FREQUENCIES && regs_cache[i]; i++) {122 do_emif_regdump_show(s, emif, regs_cache[i]);123 seq_putc(s, '\n');124 }125 126 return 0;127}128 129DEFINE_SHOW_ATTRIBUTE(emif_regdump);130 131static int emif_mr4_show(struct seq_file *s, void *unused)132{133 struct emif_data *emif = s->private;134 135 seq_printf(s, "MR4=%d\n", emif->temperature_level);136 return 0;137}138 139DEFINE_SHOW_ATTRIBUTE(emif_mr4);140 141static void emif_debugfs_init(struct emif_data *emif)142{143 if (IS_ENABLED(CONFIG_DEBUG_FS)) {144 emif->debugfs_root = debugfs_create_dir(dev_name(emif->dev), NULL);145 debugfs_create_file("regcache_dump", S_IRUGO, emif->debugfs_root, emif,146 &emif_regdump_fops);147 debugfs_create_file("mr4", S_IRUGO, emif->debugfs_root, emif,148 &emif_mr4_fops);149 }150}151 152static void emif_debugfs_exit(struct emif_data *emif)153{154 if (IS_ENABLED(CONFIG_DEBUG_FS)) {155 debugfs_remove_recursive(emif->debugfs_root);156 emif->debugfs_root = NULL;157 }158}159 160/*161 * Get bus width used by EMIF. Note that this may be different from the162 * bus width of the DDR devices used. For instance two 16-bit DDR devices163 * may be connected to a given CS of EMIF. In this case bus width as far164 * as EMIF is concerned is 32, where as the DDR bus width is 16 bits.165 */166static u32 get_emif_bus_width(struct emif_data *emif)167{168 u32 width;169 void __iomem *base = emif->base;170 171 width = (readl(base + EMIF_SDRAM_CONFIG) & NARROW_MODE_MASK)172 >> NARROW_MODE_SHIFT;173 width = width == 0 ? 32 : 16;174 175 return width;176}177 178static void set_lpmode(struct emif_data *emif, u8 lpmode)179{180 u32 temp;181 void __iomem *base = emif->base;182 183 /*184 * Workaround for errata i743 - LPDDR2 Power-Down State is Not185 * Efficient186 *187 * i743 DESCRIPTION:188 * The EMIF supports power-down state for low power. The EMIF189 * automatically puts the SDRAM into power-down after the memory is190 * not accessed for a defined number of cycles and the191 * EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE bit field is set to 0x4.192 * As the EMIF supports automatic output impedance calibration, a ZQ193 * calibration long command is issued every time it exits active194 * power-down and precharge power-down modes. The EMIF waits and195 * blocks any other command during this calibration.196 * The EMIF does not allow selective disabling of ZQ calibration upon197 * exit of power-down mode. Due to very short periods of power-down198 * cycles, ZQ calibration overhead creates bandwidth issues and199 * increases overall system power consumption. On the other hand,200 * issuing ZQ calibration long commands when exiting self-refresh is201 * still required.202 *203 * WORKAROUND204 * Because there is no power consumption benefit of the power-down due205 * to the calibration and there is a performance risk, the guideline206 * is to not allow power-down state and, therefore, to not have set207 * the EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE bit field to 0x4.208 */209 if ((emif->plat_data->ip_rev == EMIF_4D) &&210 (lpmode == EMIF_LP_MODE_PWR_DN)) {211 WARN_ONCE(1,212 "REG_LP_MODE = LP_MODE_PWR_DN(4) is prohibited by erratum i743 switch to LP_MODE_SELF_REFRESH(2)\n");213 /* rollback LP_MODE to Self-refresh mode */214 lpmode = EMIF_LP_MODE_SELF_REFRESH;215 }216 217 temp = readl(base + EMIF_POWER_MANAGEMENT_CONTROL);218 temp &= ~LP_MODE_MASK;219 temp |= (lpmode << LP_MODE_SHIFT);220 writel(temp, base + EMIF_POWER_MANAGEMENT_CONTROL);221}222 223static void do_freq_update(void)224{225 struct emif_data *emif;226 227 /*228 * Workaround for errata i728: Disable LPMODE during FREQ_UPDATE229 *230 * i728 DESCRIPTION:231 * The EMIF automatically puts the SDRAM into self-refresh mode232 * after the EMIF has not performed accesses during233 * EMIF_PWR_MGMT_CTRL[7:4] REG_SR_TIM number of DDR clock cycles234 * and the EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE bit field is set235 * to 0x2. If during a small window the following three events236 * occur:237 * - The SR_TIMING counter expires238 * - And frequency change is requested239 * - And OCP access is requested240 * Then it causes instable clock on the DDR interface.241 *242 * WORKAROUND243 * To avoid the occurrence of the three events, the workaround244 * is to disable the self-refresh when requesting a frequency245 * change. Before requesting a frequency change the software must246 * program EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE to 0x0. When the247 * frequency change has been done, the software can reprogram248 * EMIF_PWR_MGMT_CTRL[10:8] REG_LP_MODE to 0x2249 */250 list_for_each_entry(emif, &device_list, node) {251 if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH)252 set_lpmode(emif, EMIF_LP_MODE_DISABLE);253 }254 255 /*256 * TODO: Do FREQ_UPDATE here when an API257 * is available for this as part of the new258 * clock framework259 */260 261 list_for_each_entry(emif, &device_list, node) {262 if (emif->lpmode == EMIF_LP_MODE_SELF_REFRESH)263 set_lpmode(emif, EMIF_LP_MODE_SELF_REFRESH);264 }265}266 267/* Find addressing table entry based on the device's type and density */268static const struct lpddr2_addressing *get_addressing_table(269 const struct ddr_device_info *device_info)270{271 u32 index, type, density;272 273 type = device_info->type;274 density = device_info->density;275 276 switch (type) {277 case DDR_TYPE_LPDDR2_S4:278 index = density - 1;279 break;280 case DDR_TYPE_LPDDR2_S2:281 switch (density) {282 case DDR_DENSITY_1Gb:283 case DDR_DENSITY_2Gb:284 index = density + 3;285 break;286 default:287 index = density - 1;288 }289 break;290 default:291 return NULL;292 }293 294 return &lpddr2_jedec_addressing_table[index];295}296 297static u32 get_zq_config_reg(const struct lpddr2_addressing *addressing,298 bool cs1_used, bool cal_resistors_per_cs)299{300 u32 zq = 0, val = 0;301 302 val = EMIF_ZQCS_INTERVAL_US * 1000 / addressing->tREFI_ns;303 zq |= val << ZQ_REFINTERVAL_SHIFT;304 305 val = DIV_ROUND_UP(T_ZQCL_DEFAULT_NS, T_ZQCS_DEFAULT_NS) - 1;306 zq |= val << ZQ_ZQCL_MULT_SHIFT;307 308 val = DIV_ROUND_UP(T_ZQINIT_DEFAULT_NS, T_ZQCL_DEFAULT_NS) - 1;309 zq |= val << ZQ_ZQINIT_MULT_SHIFT;310 311 zq |= ZQ_SFEXITEN_ENABLE << ZQ_SFEXITEN_SHIFT;312 313 if (cal_resistors_per_cs)314 zq |= ZQ_DUALCALEN_ENABLE << ZQ_DUALCALEN_SHIFT;315 else316 zq |= ZQ_DUALCALEN_DISABLE << ZQ_DUALCALEN_SHIFT;317 318 zq |= ZQ_CS0EN_MASK; /* CS0 is used for sure */319 320 val = cs1_used ? 1 : 0;321 zq |= val << ZQ_CS1EN_SHIFT;322 323 return zq;324}325 326static u32 get_temp_alert_config(const struct lpddr2_addressing *addressing,327 const struct emif_custom_configs *custom_configs, bool cs1_used,328 u32 sdram_io_width, u32 emif_bus_width)329{330 u32 alert = 0, interval, devcnt;331 332 if (custom_configs && (custom_configs->mask &333 EMIF_CUSTOM_CONFIG_TEMP_ALERT_POLL_INTERVAL))334 interval = custom_configs->temp_alert_poll_interval_ms;335 else336 interval = TEMP_ALERT_POLL_INTERVAL_DEFAULT_MS;337 338 interval *= 1000000; /* Convert to ns */339 interval /= addressing->tREFI_ns; /* Convert to refresh cycles */340 alert |= (interval << TA_REFINTERVAL_SHIFT);341 342 /*343 * sdram_io_width is in 'log2(x) - 1' form. Convert emif_bus_width344 * also to this form and subtract to get TA_DEVCNT, which is345 * in log2(x) form.346 */347 emif_bus_width = __fls(emif_bus_width) - 1;348 devcnt = emif_bus_width - sdram_io_width;349 alert |= devcnt << TA_DEVCNT_SHIFT;350 351 /* DEVWDT is in 'log2(x) - 3' form */352 alert |= (sdram_io_width - 2) << TA_DEVWDT_SHIFT;353 354 alert |= 1 << TA_SFEXITEN_SHIFT;355 alert |= 1 << TA_CS0EN_SHIFT;356 alert |= (cs1_used ? 1 : 0) << TA_CS1EN_SHIFT;357 358 return alert;359}360 361static u32 get_pwr_mgmt_ctrl(u32 freq, struct emif_data *emif, u32 ip_rev)362{363 u32 pwr_mgmt_ctrl = 0, timeout;364 u32 lpmode = EMIF_LP_MODE_SELF_REFRESH;365 u32 timeout_perf = EMIF_LP_MODE_TIMEOUT_PERFORMANCE;366 u32 timeout_pwr = EMIF_LP_MODE_TIMEOUT_POWER;367 u32 freq_threshold = EMIF_LP_MODE_FREQ_THRESHOLD;368 u32 mask;369 u8 shift;370 371 struct emif_custom_configs *cust_cfgs = emif->plat_data->custom_configs;372 373 if (cust_cfgs && (cust_cfgs->mask & EMIF_CUSTOM_CONFIG_LPMODE)) {374 lpmode = cust_cfgs->lpmode;375 timeout_perf = cust_cfgs->lpmode_timeout_performance;376 timeout_pwr = cust_cfgs->lpmode_timeout_power;377 freq_threshold = cust_cfgs->lpmode_freq_threshold;378 }379 380 /* Timeout based on DDR frequency */381 timeout = freq >= freq_threshold ? timeout_perf : timeout_pwr;382 383 /*384 * The value to be set in register is "log2(timeout) - 3"385 * if timeout < 16 load 0 in register386 * if timeout is not a power of 2, round to next highest power of 2387 */388 if (timeout < 16) {389 timeout = 0;390 } else {391 if (timeout & (timeout - 1))392 timeout <<= 1;393 timeout = __fls(timeout) - 3;394 }395 396 switch (lpmode) {397 case EMIF_LP_MODE_CLOCK_STOP:398 shift = CS_TIM_SHIFT;399 mask = CS_TIM_MASK;400 break;401 case EMIF_LP_MODE_SELF_REFRESH:402 /* Workaround for errata i735 */403 if (timeout < 6)404 timeout = 6;405 406 shift = SR_TIM_SHIFT;407 mask = SR_TIM_MASK;408 break;409 case EMIF_LP_MODE_PWR_DN:410 shift = PD_TIM_SHIFT;411 mask = PD_TIM_MASK;412 break;413 case EMIF_LP_MODE_DISABLE:414 default:415 mask = 0;416 shift = 0;417 break;418 }419 /* Round to maximum in case of overflow, BUT warn! */420 if (lpmode != EMIF_LP_MODE_DISABLE && timeout > mask >> shift) {421 pr_err("TIMEOUT Overflow - lpmode=%d perf=%d pwr=%d freq=%d\n",422 lpmode,423 timeout_perf,424 timeout_pwr,425 freq_threshold);426 WARN(1, "timeout=0x%02x greater than 0x%02x. Using max\n",427 timeout, mask >> shift);428 timeout = mask >> shift;429 }430 431 /* Setup required timing */432 pwr_mgmt_ctrl = (timeout << shift) & mask;433 /* setup a default mask for rest of the modes */434 pwr_mgmt_ctrl |= (SR_TIM_MASK | CS_TIM_MASK | PD_TIM_MASK) &435 ~mask;436 437 /* No CS_TIM in EMIF_4D5 */438 if (ip_rev == EMIF_4D5)439 pwr_mgmt_ctrl &= ~CS_TIM_MASK;440 441 pwr_mgmt_ctrl |= lpmode << LP_MODE_SHIFT;442 443 return pwr_mgmt_ctrl;444}445 446/*447 * Get the temperature level of the EMIF instance:448 * Reads the MR4 register of attached SDRAM parts to find out the temperature449 * level. If there are two parts attached(one on each CS), then the temperature450 * level for the EMIF instance is the higher of the two temperatures.451 */452static void get_temperature_level(struct emif_data *emif)453{454 u32 temp, temperature_level;455 void __iomem *base;456 457 base = emif->base;458 459 /* Read mode register 4 */460 writel(DDR_MR4, base + EMIF_LPDDR2_MODE_REG_CONFIG);461 temperature_level = readl(base + EMIF_LPDDR2_MODE_REG_DATA);462 temperature_level = (temperature_level & MR4_SDRAM_REF_RATE_MASK) >>463 MR4_SDRAM_REF_RATE_SHIFT;464 465 if (emif->plat_data->device_info->cs1_used) {466 writel(DDR_MR4 | CS_MASK, base + EMIF_LPDDR2_MODE_REG_CONFIG);467 temp = readl(base + EMIF_LPDDR2_MODE_REG_DATA);468 temp = (temp & MR4_SDRAM_REF_RATE_MASK)469 >> MR4_SDRAM_REF_RATE_SHIFT;470 temperature_level = max(temp, temperature_level);471 }472 473 /* treat everything less than nominal(3) in MR4 as nominal */474 if (unlikely(temperature_level < SDRAM_TEMP_NOMINAL))475 temperature_level = SDRAM_TEMP_NOMINAL;476 477 /* if we get reserved value in MR4 persist with the existing value */478 if (likely(temperature_level != SDRAM_TEMP_RESERVED_4))479 emif->temperature_level = temperature_level;480}481 482/*483 * setup_temperature_sensitive_regs() - set the timings for temperature484 * sensitive registers. This happens once at initialisation time based485 * on the temperature at boot time and subsequently based on the temperature486 * alert interrupt. Temperature alert can happen when the temperature487 * increases or drops. So this function can have the effect of either488 * derating the timings or going back to nominal values.489 */490static void setup_temperature_sensitive_regs(struct emif_data *emif,491 struct emif_regs *regs)492{493 u32 tim1, tim3, ref_ctrl, type;494 void __iomem *base = emif->base;495 u32 temperature;496 497 type = emif->plat_data->device_info->type;498 499 tim1 = regs->sdram_tim1_shdw;500 tim3 = regs->sdram_tim3_shdw;501 ref_ctrl = regs->ref_ctrl_shdw;502 503 /* No de-rating for non-lpddr2 devices */504 if (type != DDR_TYPE_LPDDR2_S2 && type != DDR_TYPE_LPDDR2_S4)505 goto out;506 507 temperature = emif->temperature_level;508 if (temperature == SDRAM_TEMP_HIGH_DERATE_REFRESH) {509 ref_ctrl = regs->ref_ctrl_shdw_derated;510 } else if (temperature == SDRAM_TEMP_HIGH_DERATE_REFRESH_AND_TIMINGS) {511 tim1 = regs->sdram_tim1_shdw_derated;512 tim3 = regs->sdram_tim3_shdw_derated;513 ref_ctrl = regs->ref_ctrl_shdw_derated;514 }515 516out:517 writel(tim1, base + EMIF_SDRAM_TIMING_1_SHDW);518 writel(tim3, base + EMIF_SDRAM_TIMING_3_SHDW);519 writel(ref_ctrl, base + EMIF_SDRAM_REFRESH_CTRL_SHDW);520}521 522static irqreturn_t handle_temp_alert(void __iomem *base, struct emif_data *emif)523{524 u32 old_temp_level;525 irqreturn_t ret;526 struct emif_custom_configs *custom_configs;527 528 guard(spinlock_irqsave)(&emif_lock);529 old_temp_level = emif->temperature_level;530 get_temperature_level(emif);531 532 if (unlikely(emif->temperature_level == old_temp_level)) {533 return IRQ_HANDLED;534 } else if (!emif->curr_regs) {535 dev_err(emif->dev, "temperature alert before registers are calculated, not de-rating timings\n");536 return IRQ_HANDLED;537 }538 539 custom_configs = emif->plat_data->custom_configs;540 541 /*542 * IF we detect higher than "nominal rating" from DDR sensor543 * on an unsupported DDR part, shutdown system544 */545 if (custom_configs && !(custom_configs->mask &546 EMIF_CUSTOM_CONFIG_EXTENDED_TEMP_PART)) {547 if (emif->temperature_level >= SDRAM_TEMP_HIGH_DERATE_REFRESH) {548 dev_err(emif->dev,549 "%s:NOT Extended temperature capable memory. Converting MR4=0x%02x as shutdown event\n",550 __func__, emif->temperature_level);551 /*552 * Temperature far too high - do kernel_power_off()553 * from thread context554 */555 emif->temperature_level = SDRAM_TEMP_VERY_HIGH_SHUTDOWN;556 return IRQ_WAKE_THREAD;557 }558 }559 560 if (emif->temperature_level < old_temp_level ||561 emif->temperature_level == SDRAM_TEMP_VERY_HIGH_SHUTDOWN) {562 /*563 * Temperature coming down - defer handling to thread OR564 * Temperature far too high - do kernel_power_off() from565 * thread context566 */567 ret = IRQ_WAKE_THREAD;568 } else {569 /* Temperature is going up - handle immediately */570 setup_temperature_sensitive_regs(emif, emif->curr_regs);571 do_freq_update();572 ret = IRQ_HANDLED;573 }574 575 return ret;576}577 578static irqreturn_t emif_interrupt_handler(int irq, void *dev_id)579{580 u32 interrupts;581 struct emif_data *emif = dev_id;582 void __iomem *base = emif->base;583 struct device *dev = emif->dev;584 irqreturn_t ret = IRQ_HANDLED;585 586 /* Save the status and clear it */587 interrupts = readl(base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS);588 writel(interrupts, base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS);589 590 /*591 * Handle temperature alert592 * Temperature alert should be same for all ports593 * So, it's enough to process it only for one of the ports594 */595 if (interrupts & TA_SYS_MASK)596 ret = handle_temp_alert(base, emif);597 598 if (interrupts & ERR_SYS_MASK)599 dev_err(dev, "Access error from SYS port - %x\n", interrupts);600 601 if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE) {602 /* Save the status and clear it */603 interrupts = readl(base + EMIF_LL_OCP_INTERRUPT_STATUS);604 writel(interrupts, base + EMIF_LL_OCP_INTERRUPT_STATUS);605 606 if (interrupts & ERR_LL_MASK)607 dev_err(dev, "Access error from LL port - %x\n",608 interrupts);609 }610 611 return ret;612}613 614static irqreturn_t emif_threaded_isr(int irq, void *dev_id)615{616 struct emif_data *emif = dev_id;617 unsigned long irq_state;618 619 if (emif->temperature_level == SDRAM_TEMP_VERY_HIGH_SHUTDOWN) {620 dev_emerg(emif->dev, "SDRAM temperature exceeds operating limit.. Needs shut down!!!\n");621 622 /* If we have Power OFF ability, use it, else try restarting */623 if (kernel_can_power_off()) {624 kernel_power_off();625 } else {626 WARN(1, "FIXME: NO pm_power_off!!! trying restart\n");627 kernel_restart("SDRAM Over-temp Emergency restart");628 }629 return IRQ_HANDLED;630 }631 632 spin_lock_irqsave(&emif_lock, irq_state);633 634 if (emif->curr_regs) {635 setup_temperature_sensitive_regs(emif, emif->curr_regs);636 do_freq_update();637 } else {638 dev_err(emif->dev, "temperature alert before registers are calculated, not de-rating timings\n");639 }640 641 spin_unlock_irqrestore(&emif_lock, irq_state);642 643 return IRQ_HANDLED;644}645 646static void clear_all_interrupts(struct emif_data *emif)647{648 void __iomem *base = emif->base;649 650 writel(readl(base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS),651 base + EMIF_SYSTEM_OCP_INTERRUPT_STATUS);652 if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE)653 writel(readl(base + EMIF_LL_OCP_INTERRUPT_STATUS),654 base + EMIF_LL_OCP_INTERRUPT_STATUS);655}656 657static void disable_and_clear_all_interrupts(struct emif_data *emif)658{659 void __iomem *base = emif->base;660 661 /* Disable all interrupts */662 writel(readl(base + EMIF_SYSTEM_OCP_INTERRUPT_ENABLE_SET),663 base + EMIF_SYSTEM_OCP_INTERRUPT_ENABLE_CLEAR);664 if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE)665 writel(readl(base + EMIF_LL_OCP_INTERRUPT_ENABLE_SET),666 base + EMIF_LL_OCP_INTERRUPT_ENABLE_CLEAR);667 668 /* Clear all interrupts */669 clear_all_interrupts(emif);670}671 672static int setup_interrupts(struct emif_data *emif, u32 irq)673{674 u32 interrupts, type;675 void __iomem *base = emif->base;676 677 type = emif->plat_data->device_info->type;678 679 clear_all_interrupts(emif);680 681 /* Enable interrupts for SYS interface */682 interrupts = EN_ERR_SYS_MASK;683 if (type == DDR_TYPE_LPDDR2_S2 || type == DDR_TYPE_LPDDR2_S4)684 interrupts |= EN_TA_SYS_MASK;685 writel(interrupts, base + EMIF_SYSTEM_OCP_INTERRUPT_ENABLE_SET);686 687 /* Enable interrupts for LL interface */688 if (emif->plat_data->hw_caps & EMIF_HW_CAPS_LL_INTERFACE) {689 /* TA need not be enabled for LL */690 interrupts = EN_ERR_LL_MASK;691 writel(interrupts, base + EMIF_LL_OCP_INTERRUPT_ENABLE_SET);692 }693 694 /* setup IRQ handlers */695 return devm_request_threaded_irq(emif->dev, irq,696 emif_interrupt_handler,697 emif_threaded_isr,698 0, dev_name(emif->dev),699 emif);700 701}702 703static void emif_onetime_settings(struct emif_data *emif)704{705 u32 pwr_mgmt_ctrl, zq, temp_alert_cfg;706 void __iomem *base = emif->base;707 const struct lpddr2_addressing *addressing;708 const struct ddr_device_info *device_info;709 710 device_info = emif->plat_data->device_info;711 addressing = get_addressing_table(device_info);712 713 /*714 * Init power management settings715 * We don't know the frequency yet. Use a high frequency716 * value for a conservative timeout setting717 */718 pwr_mgmt_ctrl = get_pwr_mgmt_ctrl(1000000000, emif,719 emif->plat_data->ip_rev);720 emif->lpmode = (pwr_mgmt_ctrl & LP_MODE_MASK) >> LP_MODE_SHIFT;721 writel(pwr_mgmt_ctrl, base + EMIF_POWER_MANAGEMENT_CONTROL);722 723 /* Init ZQ calibration settings */724 zq = get_zq_config_reg(addressing, device_info->cs1_used,725 device_info->cal_resistors_per_cs);726 writel(zq, base + EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG);727 728 /* Check temperature level temperature level*/729 get_temperature_level(emif);730 if (emif->temperature_level == SDRAM_TEMP_VERY_HIGH_SHUTDOWN)731 dev_emerg(emif->dev, "SDRAM temperature exceeds operating limit.. Needs shut down!!!\n");732 733 /* Init temperature polling */734 temp_alert_cfg = get_temp_alert_config(addressing,735 emif->plat_data->custom_configs, device_info->cs1_used,736 device_info->io_width, get_emif_bus_width(emif));737 writel(temp_alert_cfg, base + EMIF_TEMPERATURE_ALERT_CONFIG);738 739 /*740 * Program external PHY control registers that are not frequency741 * dependent742 */743 if (emif->plat_data->phy_type != EMIF_PHY_TYPE_INTELLIPHY)744 return;745 writel(EMIF_EXT_PHY_CTRL_1_VAL, base + EMIF_EXT_PHY_CTRL_1_SHDW);746 writel(EMIF_EXT_PHY_CTRL_5_VAL, base + EMIF_EXT_PHY_CTRL_5_SHDW);747 writel(EMIF_EXT_PHY_CTRL_6_VAL, base + EMIF_EXT_PHY_CTRL_6_SHDW);748 writel(EMIF_EXT_PHY_CTRL_7_VAL, base + EMIF_EXT_PHY_CTRL_7_SHDW);749 writel(EMIF_EXT_PHY_CTRL_8_VAL, base + EMIF_EXT_PHY_CTRL_8_SHDW);750 writel(EMIF_EXT_PHY_CTRL_9_VAL, base + EMIF_EXT_PHY_CTRL_9_SHDW);751 writel(EMIF_EXT_PHY_CTRL_10_VAL, base + EMIF_EXT_PHY_CTRL_10_SHDW);752 writel(EMIF_EXT_PHY_CTRL_11_VAL, base + EMIF_EXT_PHY_CTRL_11_SHDW);753 writel(EMIF_EXT_PHY_CTRL_12_VAL, base + EMIF_EXT_PHY_CTRL_12_SHDW);754 writel(EMIF_EXT_PHY_CTRL_13_VAL, base + EMIF_EXT_PHY_CTRL_13_SHDW);755 writel(EMIF_EXT_PHY_CTRL_14_VAL, base + EMIF_EXT_PHY_CTRL_14_SHDW);756 writel(EMIF_EXT_PHY_CTRL_15_VAL, base + EMIF_EXT_PHY_CTRL_15_SHDW);757 writel(EMIF_EXT_PHY_CTRL_16_VAL, base + EMIF_EXT_PHY_CTRL_16_SHDW);758 writel(EMIF_EXT_PHY_CTRL_17_VAL, base + EMIF_EXT_PHY_CTRL_17_SHDW);759 writel(EMIF_EXT_PHY_CTRL_18_VAL, base + EMIF_EXT_PHY_CTRL_18_SHDW);760 writel(EMIF_EXT_PHY_CTRL_19_VAL, base + EMIF_EXT_PHY_CTRL_19_SHDW);761 writel(EMIF_EXT_PHY_CTRL_20_VAL, base + EMIF_EXT_PHY_CTRL_20_SHDW);762 writel(EMIF_EXT_PHY_CTRL_21_VAL, base + EMIF_EXT_PHY_CTRL_21_SHDW);763 writel(EMIF_EXT_PHY_CTRL_22_VAL, base + EMIF_EXT_PHY_CTRL_22_SHDW);764 writel(EMIF_EXT_PHY_CTRL_23_VAL, base + EMIF_EXT_PHY_CTRL_23_SHDW);765 writel(EMIF_EXT_PHY_CTRL_24_VAL, base + EMIF_EXT_PHY_CTRL_24_SHDW);766}767 768static void get_default_timings(struct emif_data *emif)769{770 struct emif_platform_data *pd = emif->plat_data;771 772 pd->timings = lpddr2_jedec_timings;773 pd->timings_arr_size = ARRAY_SIZE(lpddr2_jedec_timings);774 775 dev_warn(emif->dev, "%s: using default timings\n", __func__);776}777 778static int is_dev_data_valid(u32 type, u32 density, u32 io_width, u32 phy_type,779 u32 ip_rev, struct device *dev)780{781 int valid;782 783 valid = (type == DDR_TYPE_LPDDR2_S4 ||784 type == DDR_TYPE_LPDDR2_S2)785 && (density >= DDR_DENSITY_64Mb786 && density <= DDR_DENSITY_8Gb)787 && (io_width >= DDR_IO_WIDTH_8788 && io_width <= DDR_IO_WIDTH_32);789 790 /* Combinations of EMIF and PHY revisions that we support today */791 switch (ip_rev) {792 case EMIF_4D:793 valid = valid && (phy_type == EMIF_PHY_TYPE_ATTILAPHY);794 break;795 case EMIF_4D5:796 valid = valid && (phy_type == EMIF_PHY_TYPE_INTELLIPHY);797 break;798 default:799 valid = 0;800 }801 802 if (!valid)803 dev_err(dev, "%s: invalid DDR details\n", __func__);804 return valid;805}806 807static int is_custom_config_valid(struct emif_custom_configs *cust_cfgs,808 struct device *dev)809{810 int valid = 1;811 812 if ((cust_cfgs->mask & EMIF_CUSTOM_CONFIG_LPMODE) &&813 (cust_cfgs->lpmode != EMIF_LP_MODE_DISABLE))814 valid = cust_cfgs->lpmode_freq_threshold &&815 cust_cfgs->lpmode_timeout_performance &&816 cust_cfgs->lpmode_timeout_power;817 818 if (cust_cfgs->mask & EMIF_CUSTOM_CONFIG_TEMP_ALERT_POLL_INTERVAL)819 valid = valid && cust_cfgs->temp_alert_poll_interval_ms;820 821 if (!valid)822 dev_warn(dev, "%s: invalid custom configs\n", __func__);823 824 return valid;825}826 827static void of_get_custom_configs(struct device_node *np_emif,828 struct emif_data *emif)829{830 struct emif_custom_configs *cust_cfgs = NULL;831 int len;832 const __be32 *lpmode, *poll_intvl;833 834 lpmode = of_get_property(np_emif, "low-power-mode", &len);835 poll_intvl = of_get_property(np_emif, "temp-alert-poll-interval", &len);836 837 if (lpmode || poll_intvl)838 cust_cfgs = devm_kzalloc(emif->dev, sizeof(*cust_cfgs),839 GFP_KERNEL);840 841 if (!cust_cfgs)842 return;843 844 if (lpmode) {845 cust_cfgs->mask |= EMIF_CUSTOM_CONFIG_LPMODE;846 cust_cfgs->lpmode = be32_to_cpup(lpmode);847 of_property_read_u32(np_emif,848 "low-power-mode-timeout-performance",849 &cust_cfgs->lpmode_timeout_performance);850 of_property_read_u32(np_emif,851 "low-power-mode-timeout-power",852 &cust_cfgs->lpmode_timeout_power);853 of_property_read_u32(np_emif,854 "low-power-mode-freq-threshold",855 &cust_cfgs->lpmode_freq_threshold);856 }857 858 if (poll_intvl) {859 cust_cfgs->mask |=860 EMIF_CUSTOM_CONFIG_TEMP_ALERT_POLL_INTERVAL;861 cust_cfgs->temp_alert_poll_interval_ms =862 be32_to_cpup(poll_intvl);863 }864 865 if (of_property_read_bool(np_emif, "extended-temp-part"))866 cust_cfgs->mask |= EMIF_CUSTOM_CONFIG_EXTENDED_TEMP_PART;867 868 if (!is_custom_config_valid(cust_cfgs, emif->dev)) {869 devm_kfree(emif->dev, cust_cfgs);870 return;871 }872 873 emif->plat_data->custom_configs = cust_cfgs;874}875 876static void of_get_ddr_info(struct device_node *np_emif,877 struct device_node *np_ddr,878 struct ddr_device_info *dev_info)879{880 u32 density = 0, io_width = 0;881 882 dev_info->cs1_used = of_property_read_bool(np_emif, "cs1-used");883 dev_info->cal_resistors_per_cs = of_property_read_bool(np_emif, "cal-resistor-per-cs");884 885 if (of_device_is_compatible(np_ddr, "jedec,lpddr2-s4"))886 dev_info->type = DDR_TYPE_LPDDR2_S4;887 else if (of_device_is_compatible(np_ddr, "jedec,lpddr2-s2"))888 dev_info->type = DDR_TYPE_LPDDR2_S2;889 890 of_property_read_u32(np_ddr, "density", &density);891 of_property_read_u32(np_ddr, "io-width", &io_width);892 893 /* Convert from density in Mb to the density encoding in jedc_ddr.h */894 if (density & (density - 1))895 dev_info->density = 0;896 else897 dev_info->density = __fls(density) - 5;898 899 /* Convert from io_width in bits to io_width encoding in jedc_ddr.h */900 if (io_width & (io_width - 1))901 dev_info->io_width = 0;902 else903 dev_info->io_width = __fls(io_width) - 1;904}905 906static struct emif_data *of_get_memory_device_details(907 struct device_node *np_emif, struct device *dev)908{909 struct emif_data *emif = NULL;910 struct ddr_device_info *dev_info = NULL;911 struct emif_platform_data *pd = NULL;912 struct device_node *np_ddr;913 914 np_ddr = of_parse_phandle(np_emif, "device-handle", 0);915 if (!np_ddr)916 goto error;917 emif = devm_kzalloc(dev, sizeof(struct emif_data), GFP_KERNEL);918 pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);919 dev_info = devm_kzalloc(dev, sizeof(*dev_info), GFP_KERNEL);920 921 if (!emif || !pd || !dev_info) {922 dev_err(dev, "%s: Out of memory!!\n",923 __func__);924 goto error;925 }926 927 emif->plat_data = pd;928 pd->device_info = dev_info;929 emif->dev = dev;930 emif->np_ddr = np_ddr;931 emif->temperature_level = SDRAM_TEMP_NOMINAL;932 933 if (of_device_is_compatible(np_emif, "ti,emif-4d"))934 emif->plat_data->ip_rev = EMIF_4D;935 else if (of_device_is_compatible(np_emif, "ti,emif-4d5"))936 emif->plat_data->ip_rev = EMIF_4D5;937 938 of_property_read_u32(np_emif, "phy-type", &pd->phy_type);939 940 if (of_property_read_bool(np_emif, "hw-caps-ll-interface"))941 pd->hw_caps |= EMIF_HW_CAPS_LL_INTERFACE;942 943 of_get_ddr_info(np_emif, np_ddr, dev_info);944 if (!is_dev_data_valid(pd->device_info->type, pd->device_info->density,945 pd->device_info->io_width, pd->phy_type, pd->ip_rev,946 emif->dev)) {947 dev_err(dev, "%s: invalid device data!!\n", __func__);948 goto error;949 }950 /*951 * For EMIF instances other than EMIF1 see if the devices connected952 * are exactly same as on EMIF1(which is typically the case). If so,953 * mark it as a duplicate of EMIF1. This will save some memory and954 * computation.955 */956 if (emif1 && emif1->np_ddr == np_ddr) {957 emif->duplicate = true;958 goto out;959 } else if (emif1) {960 dev_warn(emif->dev, "%s: Non-symmetric DDR geometry\n",961 __func__);962 }963 964 of_get_custom_configs(np_emif, emif);965 emif->plat_data->timings = of_get_ddr_timings(np_ddr, emif->dev,966 emif->plat_data->device_info->type,967 &emif->plat_data->timings_arr_size);968 969 emif->plat_data->min_tck = of_get_min_tck(np_ddr, emif->dev);970 goto out;971 972error:973 return NULL;974out:975 return emif;976}977 978static struct emif_data *get_device_details(979 struct platform_device *pdev)980{981 u32 size;982 struct emif_data *emif = NULL;983 struct ddr_device_info *dev_info;984 struct emif_custom_configs *cust_cfgs;985 struct emif_platform_data *pd;986 struct device *dev;987 void *temp;988 989 pd = pdev->dev.platform_data;990 dev = &pdev->dev;991 992 if (!(pd && pd->device_info && is_dev_data_valid(pd->device_info->type,993 pd->device_info->density, pd->device_info->io_width,994 pd->phy_type, pd->ip_rev, dev))) {995 dev_err(dev, "%s: invalid device data\n", __func__);996 goto error;997 }998 999 emif = devm_kzalloc(dev, sizeof(*emif), GFP_KERNEL);1000 temp = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);1001 dev_info = devm_kzalloc(dev, sizeof(*dev_info), GFP_KERNEL);1002 1003 if (!emif || !temp || !dev_info)1004 goto error;1005 1006 memcpy(temp, pd, sizeof(*pd));1007 pd = temp;1008 memcpy(dev_info, pd->device_info, sizeof(*dev_info));1009 1010 pd->device_info = dev_info;1011 emif->plat_data = pd;1012 emif->dev = dev;1013 emif->temperature_level = SDRAM_TEMP_NOMINAL;1014 1015 /*1016 * For EMIF instances other than EMIF1 see if the devices connected1017 * are exactly same as on EMIF1(which is typically the case). If so,1018 * mark it as a duplicate of EMIF1 and skip copying timings data.1019 * This will save some memory and some computation later.1020 */1021 emif->duplicate = emif1 && (memcmp(dev_info,1022 emif1->plat_data->device_info,1023 sizeof(struct ddr_device_info)) == 0);1024 1025 if (emif->duplicate) {1026 pd->timings = NULL;1027 pd->min_tck = NULL;1028 goto out;1029 } else if (emif1) {1030 dev_warn(emif->dev, "%s: Non-symmetric DDR geometry\n",1031 __func__);1032 }1033 1034 /*1035 * Copy custom configs - ignore allocation error, if any, as1036 * custom_configs is not very critical1037 */1038 cust_cfgs = pd->custom_configs;1039 if (cust_cfgs && is_custom_config_valid(cust_cfgs, dev)) {1040 temp = devm_kzalloc(dev, sizeof(*cust_cfgs), GFP_KERNEL);1041 if (temp)1042 memcpy(temp, cust_cfgs, sizeof(*cust_cfgs));1043 pd->custom_configs = temp;1044 }1045 1046 /*1047 * Copy timings and min-tck values from platform data. If it is not1048 * available or if memory allocation fails, use JEDEC defaults1049 */1050 size = sizeof(struct lpddr2_timings) * pd->timings_arr_size;1051 if (pd->timings) {1052 temp = devm_kzalloc(dev, size, GFP_KERNEL);1053 if (temp) {1054 memcpy(temp, pd->timings, size);1055 pd->timings = temp;1056 } else {1057 get_default_timings(emif);1058 }1059 } else {1060 get_default_timings(emif);1061 }1062 1063 if (pd->min_tck) {1064 temp = devm_kzalloc(dev, sizeof(*pd->min_tck), GFP_KERNEL);1065 if (temp) {1066 memcpy(temp, pd->min_tck, sizeof(*pd->min_tck));1067 pd->min_tck = temp;1068 } else {1069 pd->min_tck = &lpddr2_jedec_min_tck;1070 }1071 } else {1072 pd->min_tck = &lpddr2_jedec_min_tck;1073 }1074 1075out:1076 return emif;1077 1078error:1079 return NULL;1080}1081 1082static int emif_probe(struct platform_device *pdev)1083{1084 struct emif_data *emif;1085 int irq, ret;1086 1087 if (pdev->dev.of_node)1088 emif = of_get_memory_device_details(pdev->dev.of_node, &pdev->dev);1089 else1090 emif = get_device_details(pdev);1091 1092 if (!emif) {1093 pr_err("%s: error getting device data\n", __func__);1094 goto error;1095 }1096 1097 list_add(&emif->node, &device_list);1098 1099 /* Save pointers to each other in emif and device structures */1100 emif->dev = &pdev->dev;1101 platform_set_drvdata(pdev, emif);1102 1103 emif->base = devm_platform_ioremap_resource(pdev, 0);1104 if (IS_ERR(emif->base))1105 goto error;1106 1107 irq = platform_get_irq(pdev, 0);1108 if (irq < 0)1109 goto error;1110 1111 emif_onetime_settings(emif);1112 emif_debugfs_init(emif);1113 disable_and_clear_all_interrupts(emif);1114 ret = setup_interrupts(emif, irq);1115 if (ret)1116 goto error;1117 1118 /* One-time actions taken on probing the first device */1119 if (!emif1) {1120 emif1 = emif;1121 1122 /*1123 * TODO: register notifiers for frequency and voltage1124 * change here once the respective frameworks are1125 * available1126 */1127 }1128 1129 dev_info(&pdev->dev, "%s: device configured with addr = %p and IRQ%d\n",1130 __func__, emif->base, irq);1131 1132 return 0;1133error:1134 return -ENODEV;1135}1136 1137static void emif_remove(struct platform_device *pdev)1138{1139 struct emif_data *emif = platform_get_drvdata(pdev);1140 1141 emif_debugfs_exit(emif);1142}1143 1144static void emif_shutdown(struct platform_device *pdev)1145{1146 struct emif_data *emif = platform_get_drvdata(pdev);1147 1148 disable_and_clear_all_interrupts(emif);1149}1150 1151#if defined(CONFIG_OF)1152static const struct of_device_id emif_of_match[] = {1153 { .compatible = "ti,emif-4d" },1154 { .compatible = "ti,emif-4d5" },1155 {},1156};1157MODULE_DEVICE_TABLE(of, emif_of_match);1158#endif1159 1160static struct platform_driver emif_driver = {1161 .probe = emif_probe,1162 .remove_new = emif_remove,1163 .shutdown = emif_shutdown,1164 .driver = {1165 .name = "emif",1166 .of_match_table = of_match_ptr(emif_of_match),1167 },1168};1169 1170module_platform_driver(emif_driver);1171 1172MODULE_DESCRIPTION("TI EMIF SDRAM Controller Driver");1173MODULE_LICENSE("GPL");1174MODULE_ALIAS("platform:emif");1175MODULE_AUTHOR("Texas Instruments Inc");1176