526 lines · c
1/*2 * AMD64 class Memory Controller kernel module3 *4 * Copyright (c) 2009 SoftwareBitMaker.5 * Copyright (c) 2009-15 Advanced Micro Devices, Inc.6 *7 * This file may be distributed under the terms of the8 * GNU General Public License.9 */10 11#include <linux/module.h>12#include <linux/ctype.h>13#include <linux/init.h>14#include <linux/pci.h>15#include <linux/pci_ids.h>16#include <linux/slab.h>17#include <linux/mmzone.h>18#include <linux/edac.h>19#include <linux/bitfield.h>20#include <asm/cpu_device_id.h>21#include <asm/msr.h>22#include "edac_module.h"23#include "mce_amd.h"24 25#define amd64_info(fmt, arg...) \26 edac_printk(KERN_INFO, "amd64", fmt, ##arg)27 28#define amd64_warn(fmt, arg...) \29 edac_printk(KERN_WARNING, "amd64", "Warning: " fmt, ##arg)30 31#define amd64_err(fmt, arg...) \32 edac_printk(KERN_ERR, "amd64", "Error: " fmt, ##arg)33 34#define amd64_mc_warn(mci, fmt, arg...) \35 edac_mc_chipset_printk(mci, KERN_WARNING, "amd64", fmt, ##arg)36 37#define amd64_mc_err(mci, fmt, arg...) \38 edac_mc_chipset_printk(mci, KERN_ERR, "amd64", fmt, ##arg)39 40/*41 * Throughout the comments in this code, the following terms are used:42 *43 * SysAddr, DramAddr, and InputAddr44 *45 * These terms come directly from the amd64 documentation46 * (AMD publication #26094). They are defined as follows:47 *48 * SysAddr:49 * This is a physical address generated by a CPU core or a device50 * doing DMA. If generated by a CPU core, a SysAddr is the result of51 * a virtual to physical address translation by the CPU core's address52 * translation mechanism (MMU).53 *54 * DramAddr:55 * A DramAddr is derived from a SysAddr by subtracting an offset that56 * depends on which node the SysAddr maps to and whether the SysAddr57 * is within a range affected by memory hoisting. The DRAM Base58 * (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers59 * determine which node a SysAddr maps to.60 *61 * If the DRAM Hole Address Register (DHAR) is enabled and the SysAddr62 * is within the range of addresses specified by this register, then63 * a value x from the DHAR is subtracted from the SysAddr to produce a64 * DramAddr. Here, x represents the base address for the node that65 * the SysAddr maps to plus an offset due to memory hoisting. See66 * section 3.4.8 and the comments in amd64_get_dram_hole_info() and67 * sys_addr_to_dram_addr() below for more information.68 *69 * If the SysAddr is not affected by the DHAR then a value y is70 * subtracted from the SysAddr to produce a DramAddr. Here, y is the71 * base address for the node that the SysAddr maps to. See section72 * 3.4.4 and the comments in sys_addr_to_dram_addr() below for more73 * information.74 *75 * InputAddr:76 * A DramAddr is translated to an InputAddr before being passed to the77 * memory controller for the node that the DramAddr is associated78 * with. The memory controller then maps the InputAddr to a csrow.79 * If node interleaving is not in use, then the InputAddr has the same80 * value as the DramAddr. Otherwise, the InputAddr is produced by81 * discarding the bits used for node interleaving from the DramAddr.82 * See section 3.4.4 for more information.83 *84 * The memory controller for a given node uses its DRAM CS Base and85 * DRAM CS Mask registers to map an InputAddr to a csrow. See86 * sections 3.5.4 and 3.5.5 for more information.87 */88 89#define EDAC_MOD_STR "amd64_edac"90 91/* Extended Model from CPUID, for CPU Revision numbers */92#define K8_REV_D 193#define K8_REV_E 294#define K8_REV_F 495 96/* Hardware limit on ChipSelect rows per MC and processors per system */97#define NUM_CHIPSELECTS 898#define DRAM_RANGES 899#define NUM_CONTROLLERS 12100 101#define ON true102#define OFF false103 104/*105 * PCI-defined configuration space registers106 */107#define PCI_DEVICE_ID_AMD_15H_NB_F1 0x1601108#define PCI_DEVICE_ID_AMD_15H_NB_F2 0x1602109#define PCI_DEVICE_ID_AMD_15H_M30H_NB_F1 0x141b110#define PCI_DEVICE_ID_AMD_15H_M30H_NB_F2 0x141c111#define PCI_DEVICE_ID_AMD_15H_M60H_NB_F1 0x1571112#define PCI_DEVICE_ID_AMD_15H_M60H_NB_F2 0x1572113#define PCI_DEVICE_ID_AMD_16H_NB_F1 0x1531114#define PCI_DEVICE_ID_AMD_16H_NB_F2 0x1532115#define PCI_DEVICE_ID_AMD_16H_M30H_NB_F1 0x1581116#define PCI_DEVICE_ID_AMD_16H_M30H_NB_F2 0x1582117 118/*119 * Function 1 - Address Map120 */121#define DRAM_BASE_LO 0x40122#define DRAM_LIMIT_LO 0x44123 124/*125 * F15 M30h D18F1x2[1C:00]126 */127#define DRAM_CONT_BASE 0x200128#define DRAM_CONT_LIMIT 0x204129 130/*131 * F15 M30h D18F1x2[4C:40]132 */133#define DRAM_CONT_HIGH_OFF 0x240134 135#define dram_rw(pvt, i) ((u8)(pvt->ranges[i].base.lo & 0x3))136#define dram_intlv_sel(pvt, i) ((u8)((pvt->ranges[i].lim.lo >> 8) & 0x7))137#define dram_dst_node(pvt, i) ((u8)(pvt->ranges[i].lim.lo & 0x7))138 139#define DHAR 0xf0140#define dhar_mem_hoist_valid(pvt) ((pvt)->dhar & BIT(1))141#define dhar_base(pvt) ((pvt)->dhar & 0xff000000)142#define k8_dhar_offset(pvt) (((pvt)->dhar & 0x0000ff00) << 16)143 144 /* NOTE: Extra mask bit vs K8 */145#define f10_dhar_offset(pvt) (((pvt)->dhar & 0x0000ff80) << 16)146 147#define DCT_CFG_SEL 0x10C148 149#define DRAM_LOCAL_NODE_BASE 0x120150#define DRAM_LOCAL_NODE_LIM 0x124151 152#define DRAM_BASE_HI 0x140153#define DRAM_LIMIT_HI 0x144154 155 156/*157 * Function 2 - DRAM controller158 */159#define DCSB0 0x40160#define DCSB1 0x140161#define DCSB_CS_ENABLE BIT(0)162 163#define DCSM0 0x60164#define DCSM1 0x160165 166#define csrow_enabled(i, dct, pvt) ((pvt)->csels[(dct)].csbases[(i)] & DCSB_CS_ENABLE)167#define csrow_sec_enabled(i, dct, pvt) ((pvt)->csels[(dct)].csbases_sec[(i)] & DCSB_CS_ENABLE)168 169#define DRAM_CONTROL 0x78170 171#define DBAM0 0x80172#define DBAM1 0x180173 174/* Extract the DIMM 'type' on the i'th DIMM from the DBAM reg value passed */175#define DBAM_DIMM(i, reg) ((((reg) >> (4*(i)))) & 0xF)176 177#define DBAM_MAX_VALUE 11178 179#define DCLR0 0x90180#define DCLR1 0x190181#define REVE_WIDTH_128 BIT(16)182#define WIDTH_128 BIT(11)183 184#define DCHR0 0x94185#define DCHR1 0x194186#define DDR3_MODE BIT(8)187 188#define DCT_SEL_LO 0x110189#define dct_high_range_enabled(pvt) ((pvt)->dct_sel_lo & BIT(0))190#define dct_interleave_enabled(pvt) ((pvt)->dct_sel_lo & BIT(2))191 192#define dct_ganging_enabled(pvt) ((boot_cpu_data.x86 == 0x10) && ((pvt)->dct_sel_lo & BIT(4)))193 194#define dct_data_intlv_enabled(pvt) ((pvt)->dct_sel_lo & BIT(5))195#define dct_memory_cleared(pvt) ((pvt)->dct_sel_lo & BIT(10))196 197#define SWAP_INTLV_REG 0x10c198 199#define DCT_SEL_HI 0x114200 201#define F15H_M60H_SCRCTRL 0x1C8202 203/*204 * Function 3 - Misc Control205 */206#define NBCTL 0x40207 208#define NBCFG 0x44209#define NBCFG_CHIPKILL BIT(23)210#define NBCFG_ECC_ENABLE BIT(22)211 212/* F3x48: NBSL */213#define F10_NBSL_EXT_ERR_ECC 0x8214#define NBSL_PP_OBS 0x2215 216#define SCRCTRL 0x58217 218#define F10_ONLINE_SPARE 0xB0219#define online_spare_swap_done(pvt, c) (((pvt)->online_spare >> (1 + 2 * (c))) & 0x1)220#define online_spare_bad_dramcs(pvt, c) (((pvt)->online_spare >> (4 + 4 * (c))) & 0x7)221 222#define F10_NB_ARRAY_ADDR 0xB8223#define F10_NB_ARRAY_DRAM BIT(31)224 225/* Bits [2:1] are used to select 16-byte section within a 64-byte cacheline */226#define SET_NB_ARRAY_ADDR(section) (((section) & 0x3) << 1)227 228#define F10_NB_ARRAY_DATA 0xBC229#define F10_NB_ARR_ECC_WR_REQ BIT(17)230#define SET_NB_DRAM_INJECTION_WRITE(inj) \231 (BIT(((inj.word) & 0xF) + 20) | \232 F10_NB_ARR_ECC_WR_REQ | inj.bit_map)233#define SET_NB_DRAM_INJECTION_READ(inj) \234 (BIT(((inj.word) & 0xF) + 20) | \235 BIT(16) | inj.bit_map)236 237 238#define NBCAP 0xE8239#define NBCAP_CHIPKILL BIT(4)240#define NBCAP_SECDED BIT(3)241#define NBCAP_DCT_DUAL BIT(0)242 243#define EXT_NB_MCA_CFG 0x180244 245/* MSRs */246#define MSR_MCGCTL_NBE BIT(4)247 248/* F17h */249 250/* F0: */251#define DF_DHAR 0x104252 253/* UMC CH register offsets */254#define UMCCH_BASE_ADDR 0x0255#define UMCCH_BASE_ADDR_SEC 0x10256#define UMCCH_ADDR_MASK 0x20257#define UMCCH_ADDR_MASK_SEC 0x28258#define UMCCH_ADDR_MASK_SEC_DDR5 0x30259#define UMCCH_DIMM_CFG 0x80260#define UMCCH_DIMM_CFG_DDR5 0x90261#define UMCCH_UMC_CFG 0x100262#define UMCCH_SDP_CTRL 0x104263#define UMCCH_ECC_CTRL 0x14C264#define UMCCH_UMC_CAP_HI 0xDF4265 266/* UMC CH bitfields */267#define UMC_ECC_CHIPKILL_CAP BIT(31)268#define UMC_ECC_ENABLED BIT(30)269 270#define UMC_SDP_INIT BIT(31)271 272/* Error injection control structure */273struct error_injection {274 u32 section;275 u32 word;276 u32 bit_map;277};278 279/* low and high part of PCI config space regs */280struct reg_pair {281 u32 lo, hi;282};283 284/*285 * See F1x[1, 0][7C:40] DRAM Base/Limit Registers286 */287struct dram_range {288 struct reg_pair base;289 struct reg_pair lim;290};291 292/* A DCT chip selects collection */293struct chip_select {294 u32 csbases[NUM_CHIPSELECTS];295 u32 csbases_sec[NUM_CHIPSELECTS];296 u8 b_cnt;297 298 u32 csmasks[NUM_CHIPSELECTS];299 u32 csmasks_sec[NUM_CHIPSELECTS];300 u8 m_cnt;301};302 303struct amd64_umc {304 u32 dimm_cfg; /* DIMM Configuration reg */305 u32 umc_cfg; /* Configuration reg */306 u32 sdp_ctrl; /* SDP Control reg */307 u32 ecc_ctrl; /* DRAM ECC Control reg */308 u32 umc_cap_hi; /* Capabilities High reg */309 310 /* cache the dram_type */311 enum mem_type dram_type;312};313 314struct amd64_family_flags {315 /*316 * Indicates that the system supports the new register offsets, etc.317 * first introduced with Family 19h Model 10h.318 */319 __u64 zn_regs_v2 : 1,320 321 __reserved : 63;322};323 324struct amd64_pvt {325 struct low_ops *ops;326 327 /* pci_device handles which we utilize */328 struct pci_dev *F1, *F2, *F3;329 330 u16 mc_node_id; /* MC index of this MC node */331 u8 fam; /* CPU family */332 u8 model; /* ... model */333 u8 stepping; /* ... stepping */334 335 int ext_model; /* extended model value of this node */336 337 /* Raw registers */338 u32 dclr0; /* DRAM Configuration Low DCT0 reg */339 u32 dclr1; /* DRAM Configuration Low DCT1 reg */340 u32 dchr0; /* DRAM Configuration High DCT0 reg */341 u32 dchr1; /* DRAM Configuration High DCT1 reg */342 u32 nbcap; /* North Bridge Capabilities */343 u32 nbcfg; /* F10 North Bridge Configuration */344 u32 dhar; /* DRAM Hoist reg */345 u32 dbam0; /* DRAM Base Address Mapping reg for DCT0 */346 u32 dbam1; /* DRAM Base Address Mapping reg for DCT1 */347 348 /* one for each DCT/UMC */349 struct chip_select csels[NUM_CONTROLLERS];350 351 /* DRAM base and limit pairs F1x[78,70,68,60,58,50,48,40] */352 struct dram_range ranges[DRAM_RANGES];353 354 u64 top_mem; /* top of memory below 4GB */355 u64 top_mem2; /* top of memory above 4GB */356 357 u32 dct_sel_lo; /* DRAM Controller Select Low */358 u32 dct_sel_hi; /* DRAM Controller Select High */359 u32 online_spare; /* On-Line spare Reg */360 u32 gpu_umc_base; /* Base address used for channel selection on GPUs */361 362 /* x4, x8, or x16 syndromes in use */363 u8 ecc_sym_sz;364 365 const char *ctl_name;366 u16 f1_id, f2_id;367 /* Maximum number of memory controllers per die/node. */368 u8 max_mcs;369 370 struct amd64_family_flags flags;371 /* place to store error injection parameters prior to issue */372 struct error_injection injection;373 374 /*375 * cache the dram_type376 *377 * NOTE: Don't use this for Family 17h and later.378 * Use dram_type in struct amd64_umc instead.379 */380 enum mem_type dram_type;381 382 struct amd64_umc *umc; /* UMC registers */383};384 385enum err_codes {386 DECODE_OK = 0,387 ERR_NODE = -1,388 ERR_CSROW = -2,389 ERR_CHANNEL = -3,390 ERR_SYND = -4,391 ERR_NORM_ADDR = -5,392};393 394struct err_info {395 int err_code;396 struct mem_ctl_info *src_mci;397 int csrow;398 int channel;399 u16 syndrome;400 u32 page;401 u32 offset;402};403 404static inline u32 get_umc_base(u8 channel)405{406 /* chY: 0xY50000 */407 return 0x50000 + (channel << 20);408}409 410static inline u64 get_dram_base(struct amd64_pvt *pvt, u8 i)411{412 u64 addr = ((u64)pvt->ranges[i].base.lo & 0xffff0000) << 8;413 414 if (boot_cpu_data.x86 == 0xf)415 return addr;416 417 return (((u64)pvt->ranges[i].base.hi & 0x000000ff) << 40) | addr;418}419 420static inline u64 get_dram_limit(struct amd64_pvt *pvt, u8 i)421{422 u64 lim = (((u64)pvt->ranges[i].lim.lo & 0xffff0000) << 8) | 0x00ffffff;423 424 if (boot_cpu_data.x86 == 0xf)425 return lim;426 427 return (((u64)pvt->ranges[i].lim.hi & 0x000000ff) << 40) | lim;428}429 430static inline u16 extract_syndrome(u64 status)431{432 return ((status >> 47) & 0xff) | ((status >> 16) & 0xff00);433}434 435static inline u8 dct_sel_interleave_addr(struct amd64_pvt *pvt)436{437 if (pvt->fam == 0x15 && pvt->model >= 0x30)438 return (((pvt->dct_sel_hi >> 9) & 0x1) << 2) |439 ((pvt->dct_sel_lo >> 6) & 0x3);440 441 return ((pvt)->dct_sel_lo >> 6) & 0x3;442}443/*444 * per-node ECC settings descriptor445 */446struct ecc_settings {447 u32 old_nbctl;448 bool nbctl_valid;449 450 struct flags {451 unsigned long nb_mce_enable:1;452 unsigned long nb_ecc_prev:1;453 } flags;454};455 456/*457 * Each of the PCI Device IDs types have their own set of hardware accessor458 * functions and per device encoding/decoding logic.459 */460struct low_ops {461 void (*map_sysaddr_to_csrow)(struct mem_ctl_info *mci, u64 sys_addr,462 struct err_info *err);463 int (*dbam_to_cs)(struct amd64_pvt *pvt, u8 dct,464 unsigned int cs_mode, int cs_mask_nr);465 int (*hw_info_get)(struct amd64_pvt *pvt);466 bool (*ecc_enabled)(struct amd64_pvt *pvt);467 void (*setup_mci_misc_attrs)(struct mem_ctl_info *mci);468 void (*dump_misc_regs)(struct amd64_pvt *pvt);469 void (*get_err_info)(struct mce *m, struct err_info *err);470};471 472int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,473 u32 *val, const char *func);474int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,475 u32 val, const char *func);476 477#define amd64_read_pci_cfg(pdev, offset, val) \478 __amd64_read_pci_cfg_dword(pdev, offset, val, __func__)479 480#define amd64_write_pci_cfg(pdev, offset, val) \481 __amd64_write_pci_cfg_dword(pdev, offset, val, __func__)482 483#define to_mci(k) container_of(k, struct mem_ctl_info, dev)484 485/* Injection helpers */486static inline void disable_caches(void *dummy)487{488 write_cr0(read_cr0() | X86_CR0_CD);489 wbinvd();490}491 492static inline void enable_caches(void *dummy)493{494 write_cr0(read_cr0() & ~X86_CR0_CD);495}496 497static inline u8 dram_intlv_en(struct amd64_pvt *pvt, unsigned int i)498{499 if (pvt->fam == 0x15 && pvt->model >= 0x30) {500 u32 tmp;501 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &tmp);502 return (u8) tmp & 0xF;503 }504 return (u8) (pvt->ranges[i].base.lo >> 8) & 0x7;505}506 507static inline u8 dhar_valid(struct amd64_pvt *pvt)508{509 if (pvt->fam == 0x15 && pvt->model >= 0x30) {510 u32 tmp;511 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &tmp);512 return (tmp >> 1) & BIT(0);513 }514 return (pvt)->dhar & BIT(0);515}516 517static inline u32 dct_sel_baseaddr(struct amd64_pvt *pvt)518{519 if (pvt->fam == 0x15 && pvt->model >= 0x30) {520 u32 tmp;521 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &tmp);522 return (tmp >> 11) & 0x1FFF;523 }524 return (pvt)->dct_sel_lo & 0xFFFFF800;525}526