633 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Freescale Memory Controller kernel module4 *5 * Support Power-based SoCs including MPC85xx, MPC86xx, MPC83xx and6 * ARM-based Layerscape SoCs including LS2xxx and LS1021A. Originally7 * split out from mpc85xx_edac EDAC driver.8 *9 * Parts Copyrighted (c) 2013 by Freescale Semiconductor, Inc.10 *11 * Author: Dave Jiang <djiang@mvista.com>12 *13 * 2006-2007 (c) MontaVista Software, Inc.14 */15#include <linux/module.h>16#include <linux/init.h>17#include <linux/interrupt.h>18#include <linux/ctype.h>19#include <linux/io.h>20#include <linux/mod_devicetable.h>21#include <linux/edac.h>22#include <linux/smp.h>23#include <linux/gfp.h>24 25#include <linux/of.h>26#include <linux/of_address.h>27#include "edac_module.h"28#include "fsl_ddr_edac.h"29 30#define EDAC_MOD_STR "fsl_ddr_edac"31 32static int edac_mc_idx;33 34static u32 orig_ddr_err_disable;35static u32 orig_ddr_err_sbe;36static bool little_endian;37 38static inline u32 ddr_in32(void __iomem *addr)39{40 return little_endian ? ioread32(addr) : ioread32be(addr);41}42 43static inline void ddr_out32(void __iomem *addr, u32 value)44{45 if (little_endian)46 iowrite32(value, addr);47 else48 iowrite32be(value, addr);49}50 51#ifdef CONFIG_EDAC_DEBUG52/************************ MC SYSFS parts ***********************************/53 54#define to_mci(k) container_of(k, struct mem_ctl_info, dev)55 56static ssize_t fsl_mc_inject_data_hi_show(struct device *dev,57 struct device_attribute *mattr,58 char *data)59{60 struct mem_ctl_info *mci = to_mci(dev);61 struct fsl_mc_pdata *pdata = mci->pvt_info;62 return sprintf(data, "0x%08x",63 ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI));64}65 66static ssize_t fsl_mc_inject_data_lo_show(struct device *dev,67 struct device_attribute *mattr,68 char *data)69{70 struct mem_ctl_info *mci = to_mci(dev);71 struct fsl_mc_pdata *pdata = mci->pvt_info;72 return sprintf(data, "0x%08x",73 ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO));74}75 76static ssize_t fsl_mc_inject_ctrl_show(struct device *dev,77 struct device_attribute *mattr,78 char *data)79{80 struct mem_ctl_info *mci = to_mci(dev);81 struct fsl_mc_pdata *pdata = mci->pvt_info;82 return sprintf(data, "0x%08x",83 ddr_in32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT));84}85 86static ssize_t fsl_mc_inject_data_hi_store(struct device *dev,87 struct device_attribute *mattr,88 const char *data, size_t count)89{90 struct mem_ctl_info *mci = to_mci(dev);91 struct fsl_mc_pdata *pdata = mci->pvt_info;92 unsigned long val;93 int rc;94 95 if (isdigit(*data)) {96 rc = kstrtoul(data, 0, &val);97 if (rc)98 return rc;99 100 ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI, val);101 return count;102 }103 return 0;104}105 106static ssize_t fsl_mc_inject_data_lo_store(struct device *dev,107 struct device_attribute *mattr,108 const char *data, size_t count)109{110 struct mem_ctl_info *mci = to_mci(dev);111 struct fsl_mc_pdata *pdata = mci->pvt_info;112 unsigned long val;113 int rc;114 115 if (isdigit(*data)) {116 rc = kstrtoul(data, 0, &val);117 if (rc)118 return rc;119 120 ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO, val);121 return count;122 }123 return 0;124}125 126static ssize_t fsl_mc_inject_ctrl_store(struct device *dev,127 struct device_attribute *mattr,128 const char *data, size_t count)129{130 struct mem_ctl_info *mci = to_mci(dev);131 struct fsl_mc_pdata *pdata = mci->pvt_info;132 unsigned long val;133 int rc;134 135 if (isdigit(*data)) {136 rc = kstrtoul(data, 0, &val);137 if (rc)138 return rc;139 140 ddr_out32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT, val);141 return count;142 }143 return 0;144}145 146static DEVICE_ATTR(inject_data_hi, S_IRUGO | S_IWUSR,147 fsl_mc_inject_data_hi_show, fsl_mc_inject_data_hi_store);148static DEVICE_ATTR(inject_data_lo, S_IRUGO | S_IWUSR,149 fsl_mc_inject_data_lo_show, fsl_mc_inject_data_lo_store);150static DEVICE_ATTR(inject_ctrl, S_IRUGO | S_IWUSR,151 fsl_mc_inject_ctrl_show, fsl_mc_inject_ctrl_store);152#endif /* CONFIG_EDAC_DEBUG */153 154static struct attribute *fsl_ddr_dev_attrs[] = {155#ifdef CONFIG_EDAC_DEBUG156 &dev_attr_inject_data_hi.attr,157 &dev_attr_inject_data_lo.attr,158 &dev_attr_inject_ctrl.attr,159#endif160 NULL161};162 163ATTRIBUTE_GROUPS(fsl_ddr_dev);164 165/**************************** MC Err device ***************************/166 167/*168 * Taken from table 8-55 in the MPC8641 User's Manual and/or 9-61 in the169 * MPC8572 User's Manual. Each line represents a syndrome bit column as a170 * 64-bit value, but split into an upper and lower 32-bit chunk. The labels171 * below correspond to Freescale's manuals.172 */173static unsigned int ecc_table[16] = {174 /* MSB LSB */175 /* [0:31] [32:63] */176 0xf00fe11e, 0xc33c0ff7, /* Syndrome bit 7 */177 0x00ff00ff, 0x00fff0ff,178 0x0f0f0f0f, 0x0f0fff00,179 0x11113333, 0x7777000f,180 0x22224444, 0x8888222f,181 0x44448888, 0xffff4441,182 0x8888ffff, 0x11118882,183 0xffff1111, 0x22221114, /* Syndrome bit 0 */184};185 186/*187 * Calculate the correct ECC value for a 64-bit value specified by high:low188 */189static u8 calculate_ecc(u32 high, u32 low)190{191 u32 mask_low;192 u32 mask_high;193 int bit_cnt;194 u8 ecc = 0;195 int i;196 int j;197 198 for (i = 0; i < 8; i++) {199 mask_high = ecc_table[i * 2];200 mask_low = ecc_table[i * 2 + 1];201 bit_cnt = 0;202 203 for (j = 0; j < 32; j++) {204 if ((mask_high >> j) & 1)205 bit_cnt ^= (high >> j) & 1;206 if ((mask_low >> j) & 1)207 bit_cnt ^= (low >> j) & 1;208 }209 210 ecc |= bit_cnt << i;211 }212 213 return ecc;214}215 216/*217 * Create the syndrome code which is generated if the data line specified by218 * 'bit' failed. Eg generate an 8-bit codes seen in Table 8-55 in the MPC8641219 * User's Manual and 9-61 in the MPC8572 User's Manual.220 */221static u8 syndrome_from_bit(unsigned int bit) {222 int i;223 u8 syndrome = 0;224 225 /*226 * Cycle through the upper or lower 32-bit portion of each value in227 * ecc_table depending on if 'bit' is in the upper or lower half of228 * 64-bit data.229 */230 for (i = bit < 32; i < 16; i += 2)231 syndrome |= ((ecc_table[i] >> (bit % 32)) & 1) << (i / 2);232 233 return syndrome;234}235 236/*237 * Decode data and ecc syndrome to determine what went wrong238 * Note: This can only decode single-bit errors239 */240static void sbe_ecc_decode(u32 cap_high, u32 cap_low, u32 cap_ecc,241 int *bad_data_bit, int *bad_ecc_bit)242{243 int i;244 u8 syndrome;245 246 *bad_data_bit = -1;247 *bad_ecc_bit = -1;248 249 /*250 * Calculate the ECC of the captured data and XOR it with the captured251 * ECC to find an ECC syndrome value we can search for252 */253 syndrome = calculate_ecc(cap_high, cap_low) ^ cap_ecc;254 255 /* Check if a data line is stuck... */256 for (i = 0; i < 64; i++) {257 if (syndrome == syndrome_from_bit(i)) {258 *bad_data_bit = i;259 return;260 }261 }262 263 /* If data is correct, check ECC bits for errors... */264 for (i = 0; i < 8; i++) {265 if ((syndrome >> i) & 0x1) {266 *bad_ecc_bit = i;267 return;268 }269 }270}271 272#define make64(high, low) (((u64)(high) << 32) | (low))273 274static void fsl_mc_check(struct mem_ctl_info *mci)275{276 struct fsl_mc_pdata *pdata = mci->pvt_info;277 struct csrow_info *csrow;278 u32 bus_width;279 u32 err_detect;280 u32 syndrome;281 u64 err_addr;282 u32 pfn;283 int row_index;284 u32 cap_high;285 u32 cap_low;286 int bad_data_bit;287 int bad_ecc_bit;288 289 err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT);290 if (!err_detect)291 return;292 293 fsl_mc_printk(mci, KERN_ERR, "Err Detect Register: %#8.8x\n",294 err_detect);295 296 /* no more processing if not ECC bit errors */297 if (!(err_detect & (DDR_EDE_SBE | DDR_EDE_MBE))) {298 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect);299 return;300 }301 302 syndrome = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ECC);303 304 /* Mask off appropriate bits of syndrome based on bus width */305 bus_width = (ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG) &306 DSC_DBW_MASK) ? 32 : 64;307 if (bus_width == 64)308 syndrome &= 0xff;309 else310 syndrome &= 0xffff;311 312 err_addr = make64(313 ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_EXT_ADDRESS),314 ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ADDRESS));315 pfn = err_addr >> PAGE_SHIFT;316 317 for (row_index = 0; row_index < mci->nr_csrows; row_index++) {318 csrow = mci->csrows[row_index];319 if ((pfn >= csrow->first_page) && (pfn <= csrow->last_page))320 break;321 }322 323 cap_high = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_HI);324 cap_low = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_LO);325 326 /*327 * Analyze single-bit errors on 64-bit wide buses328 * TODO: Add support for 32-bit wide buses329 */330 if ((err_detect & DDR_EDE_SBE) && (bus_width == 64)) {331 sbe_ecc_decode(cap_high, cap_low, syndrome,332 &bad_data_bit, &bad_ecc_bit);333 334 if (bad_data_bit != -1)335 fsl_mc_printk(mci, KERN_ERR,336 "Faulty Data bit: %d\n", bad_data_bit);337 if (bad_ecc_bit != -1)338 fsl_mc_printk(mci, KERN_ERR,339 "Faulty ECC bit: %d\n", bad_ecc_bit);340 341 fsl_mc_printk(mci, KERN_ERR,342 "Expected Data / ECC:\t%#8.8x_%08x / %#2.2x\n",343 cap_high ^ (1 << (bad_data_bit - 32)),344 cap_low ^ (1 << bad_data_bit),345 syndrome ^ (1 << bad_ecc_bit));346 }347 348 fsl_mc_printk(mci, KERN_ERR,349 "Captured Data / ECC:\t%#8.8x_%08x / %#2.2x\n",350 cap_high, cap_low, syndrome);351 fsl_mc_printk(mci, KERN_ERR, "Err addr: %#8.8llx\n", err_addr);352 fsl_mc_printk(mci, KERN_ERR, "PFN: %#8.8x\n", pfn);353 354 /* we are out of range */355 if (row_index == mci->nr_csrows)356 fsl_mc_printk(mci, KERN_ERR, "PFN out of range!\n");357 358 if (err_detect & DDR_EDE_SBE)359 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,360 pfn, err_addr & ~PAGE_MASK, syndrome,361 row_index, 0, -1,362 mci->ctl_name, "");363 364 if (err_detect & DDR_EDE_MBE)365 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,366 pfn, err_addr & ~PAGE_MASK, syndrome,367 row_index, 0, -1,368 mci->ctl_name, "");369 370 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect);371}372 373static irqreturn_t fsl_mc_isr(int irq, void *dev_id)374{375 struct mem_ctl_info *mci = dev_id;376 struct fsl_mc_pdata *pdata = mci->pvt_info;377 u32 err_detect;378 379 err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT);380 if (!err_detect)381 return IRQ_NONE;382 383 fsl_mc_check(mci);384 385 return IRQ_HANDLED;386}387 388static void fsl_ddr_init_csrows(struct mem_ctl_info *mci)389{390 struct fsl_mc_pdata *pdata = mci->pvt_info;391 struct csrow_info *csrow;392 struct dimm_info *dimm;393 u32 sdram_ctl;394 u32 sdtype;395 enum mem_type mtype;396 u32 cs_bnds;397 int index;398 399 sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG);400 401 sdtype = sdram_ctl & DSC_SDTYPE_MASK;402 if (sdram_ctl & DSC_RD_EN) {403 switch (sdtype) {404 case 0x02000000:405 mtype = MEM_RDDR;406 break;407 case 0x03000000:408 mtype = MEM_RDDR2;409 break;410 case 0x07000000:411 mtype = MEM_RDDR3;412 break;413 case 0x05000000:414 mtype = MEM_RDDR4;415 break;416 default:417 mtype = MEM_UNKNOWN;418 break;419 }420 } else {421 switch (sdtype) {422 case 0x02000000:423 mtype = MEM_DDR;424 break;425 case 0x03000000:426 mtype = MEM_DDR2;427 break;428 case 0x07000000:429 mtype = MEM_DDR3;430 break;431 case 0x05000000:432 mtype = MEM_DDR4;433 break;434 default:435 mtype = MEM_UNKNOWN;436 break;437 }438 }439 440 for (index = 0; index < mci->nr_csrows; index++) {441 u32 start;442 u32 end;443 444 csrow = mci->csrows[index];445 dimm = csrow->channels[0]->dimm;446 447 cs_bnds = ddr_in32(pdata->mc_vbase + FSL_MC_CS_BNDS_0 +448 (index * FSL_MC_CS_BNDS_OFS));449 450 start = (cs_bnds & 0xffff0000) >> 16;451 end = (cs_bnds & 0x0000ffff);452 453 if (start == end)454 continue; /* not populated */455 456 start <<= (24 - PAGE_SHIFT);457 end <<= (24 - PAGE_SHIFT);458 end |= (1 << (24 - PAGE_SHIFT)) - 1;459 460 csrow->first_page = start;461 csrow->last_page = end;462 463 dimm->nr_pages = end + 1 - start;464 dimm->grain = 8;465 dimm->mtype = mtype;466 dimm->dtype = DEV_UNKNOWN;467 if (sdram_ctl & DSC_X32_EN)468 dimm->dtype = DEV_X32;469 dimm->edac_mode = EDAC_SECDED;470 }471}472 473int fsl_mc_err_probe(struct platform_device *op)474{475 struct mem_ctl_info *mci;476 struct edac_mc_layer layers[2];477 struct fsl_mc_pdata *pdata;478 struct resource r;479 u32 sdram_ctl;480 int res;481 482 if (!devres_open_group(&op->dev, fsl_mc_err_probe, GFP_KERNEL))483 return -ENOMEM;484 485 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;486 layers[0].size = 4;487 layers[0].is_virt_csrow = true;488 layers[1].type = EDAC_MC_LAYER_CHANNEL;489 layers[1].size = 1;490 layers[1].is_virt_csrow = false;491 mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers,492 sizeof(*pdata));493 if (!mci) {494 devres_release_group(&op->dev, fsl_mc_err_probe);495 return -ENOMEM;496 }497 498 pdata = mci->pvt_info;499 pdata->name = "fsl_mc_err";500 mci->pdev = &op->dev;501 pdata->edac_idx = edac_mc_idx++;502 dev_set_drvdata(mci->pdev, mci);503 mci->ctl_name = pdata->name;504 mci->dev_name = pdata->name;505 506 /*507 * Get the endianness of DDR controller registers.508 * Default is big endian.509 */510 little_endian = of_property_read_bool(op->dev.of_node, "little-endian");511 512 res = of_address_to_resource(op->dev.of_node, 0, &r);513 if (res) {514 pr_err("%s: Unable to get resource for MC err regs\n",515 __func__);516 goto err;517 }518 519 if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r),520 pdata->name)) {521 pr_err("%s: Error while requesting mem region\n",522 __func__);523 res = -EBUSY;524 goto err;525 }526 527 pdata->mc_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r));528 if (!pdata->mc_vbase) {529 pr_err("%s: Unable to setup MC err regs\n", __func__);530 res = -ENOMEM;531 goto err;532 }533 534 sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG);535 if (!(sdram_ctl & DSC_ECC_EN)) {536 /* no ECC */537 pr_warn("%s: No ECC DIMMs discovered\n", __func__);538 res = -ENODEV;539 goto err;540 }541 542 edac_dbg(3, "init mci\n");543 mci->mtype_cap = MEM_FLAG_DDR | MEM_FLAG_RDDR |544 MEM_FLAG_DDR2 | MEM_FLAG_RDDR2 |545 MEM_FLAG_DDR3 | MEM_FLAG_RDDR3 |546 MEM_FLAG_DDR4 | MEM_FLAG_RDDR4;547 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;548 mci->edac_cap = EDAC_FLAG_SECDED;549 mci->mod_name = EDAC_MOD_STR;550 551 if (edac_op_state == EDAC_OPSTATE_POLL)552 mci->edac_check = fsl_mc_check;553 554 mci->ctl_page_to_phys = NULL;555 556 mci->scrub_mode = SCRUB_SW_SRC;557 558 fsl_ddr_init_csrows(mci);559 560 /* store the original error disable bits */561 orig_ddr_err_disable = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DISABLE);562 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE, 0);563 564 /* clear all error bits */565 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, ~0);566 567 res = edac_mc_add_mc_with_groups(mci, fsl_ddr_dev_groups);568 if (res) {569 edac_dbg(3, "failed edac_mc_add_mc()\n");570 goto err;571 }572 573 if (edac_op_state == EDAC_OPSTATE_INT) {574 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN,575 DDR_EIE_MBEE | DDR_EIE_SBEE);576 577 /* store the original error management threshold */578 orig_ddr_err_sbe = ddr_in32(pdata->mc_vbase +579 FSL_MC_ERR_SBE) & 0xff0000;580 581 /* set threshold to 1 error per interrupt */582 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, 0x10000);583 584 /* register interrupts */585 pdata->irq = platform_get_irq(op, 0);586 res = devm_request_irq(&op->dev, pdata->irq,587 fsl_mc_isr,588 IRQF_SHARED,589 "[EDAC] MC err", mci);590 if (res < 0) {591 pr_err("%s: Unable to request irq %d for FSL DDR DRAM ERR\n",592 __func__, pdata->irq);593 res = -ENODEV;594 goto err2;595 }596 597 pr_info(EDAC_MOD_STR " acquired irq %d for MC\n",598 pdata->irq);599 }600 601 devres_remove_group(&op->dev, fsl_mc_err_probe);602 edac_dbg(3, "success\n");603 pr_info(EDAC_MOD_STR " MC err registered\n");604 605 return 0;606 607err2:608 edac_mc_del_mc(&op->dev);609err:610 devres_release_group(&op->dev, fsl_mc_err_probe);611 edac_mc_free(mci);612 return res;613}614 615void fsl_mc_err_remove(struct platform_device *op)616{617 struct mem_ctl_info *mci = dev_get_drvdata(&op->dev);618 struct fsl_mc_pdata *pdata = mci->pvt_info;619 620 edac_dbg(0, "\n");621 622 if (edac_op_state == EDAC_OPSTATE_INT) {623 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN, 0);624 }625 626 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE,627 orig_ddr_err_disable);628 ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, orig_ddr_err_sbe);629 630 edac_mc_del_mc(&op->dev);631 edac_mc_free(mci);632}633