1691 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * APEI Generic Hardware Error Source support4 *5 * Generic Hardware Error Source provides a way to report platform6 * hardware errors (such as that from chipset). It works in so called7 * "Firmware First" mode, that is, hardware errors are reported to8 * firmware firstly, then reported to Linux by firmware. This way,9 * some non-standard hardware error registers or non-standard hardware10 * link can be checked by firmware to produce more hardware error11 * information for Linux.12 *13 * For more information about Generic Hardware Error Source, please14 * refer to ACPI Specification version 4.0, section 17.3.2.615 *16 * Copyright 2010,2011 Intel Corp.17 * Author: Huang Ying <ying.huang@intel.com>18 */19 20#include <linux/arm_sdei.h>21#include <linux/kernel.h>22#include <linux/moduleparam.h>23#include <linux/init.h>24#include <linux/acpi.h>25#include <linux/io.h>26#include <linux/interrupt.h>27#include <linux/timer.h>28#include <linux/cper.h>29#include <linux/cleanup.h>30#include <linux/platform_device.h>31#include <linux/mutex.h>32#include <linux/ratelimit.h>33#include <linux/vmalloc.h>34#include <linux/irq_work.h>35#include <linux/llist.h>36#include <linux/genalloc.h>37#include <linux/kfifo.h>38#include <linux/pci.h>39#include <linux/pfn.h>40#include <linux/aer.h>41#include <linux/nmi.h>42#include <linux/sched/clock.h>43#include <linux/uuid.h>44#include <linux/ras.h>45#include <linux/task_work.h>46 47#include <acpi/actbl1.h>48#include <acpi/ghes.h>49#include <acpi/apei.h>50#include <asm/fixmap.h>51#include <asm/tlbflush.h>52#include <cxl/event.h>53#include <ras/ras_event.h>54 55#include "apei-internal.h"56 57#define GHES_PFX "GHES: "58 59#define GHES_ESTATUS_MAX_SIZE 6553660#define GHES_ESOURCE_PREALLOC_MAX_SIZE 6553661 62#define GHES_ESTATUS_POOL_MIN_ALLOC_ORDER 363 64/* This is just an estimation for memory pool allocation */65#define GHES_ESTATUS_CACHE_AVG_SIZE 51266 67#define GHES_ESTATUS_CACHES_SIZE 468 69#define GHES_ESTATUS_IN_CACHE_MAX_NSEC 10000000000ULL70/* Prevent too many caches are allocated because of RCU */71#define GHES_ESTATUS_CACHE_ALLOCED_MAX (GHES_ESTATUS_CACHES_SIZE * 3 / 2)72 73#define GHES_ESTATUS_CACHE_LEN(estatus_len) \74 (sizeof(struct ghes_estatus_cache) + (estatus_len))75#define GHES_ESTATUS_FROM_CACHE(estatus_cache) \76 ((struct acpi_hest_generic_status *) \77 ((struct ghes_estatus_cache *)(estatus_cache) + 1))78 79#define GHES_ESTATUS_NODE_LEN(estatus_len) \80 (sizeof(struct ghes_estatus_node) + (estatus_len))81#define GHES_ESTATUS_FROM_NODE(estatus_node) \82 ((struct acpi_hest_generic_status *) \83 ((struct ghes_estatus_node *)(estatus_node) + 1))84 85#define GHES_VENDOR_ENTRY_LEN(gdata_len) \86 (sizeof(struct ghes_vendor_record_entry) + (gdata_len))87#define GHES_GDATA_FROM_VENDOR_ENTRY(vendor_entry) \88 ((struct acpi_hest_generic_data *) \89 ((struct ghes_vendor_record_entry *)(vendor_entry) + 1))90 91/*92 * NMI-like notifications vary by architecture, before the compiler can prune93 * unused static functions it needs a value for these enums.94 */95#ifndef CONFIG_ARM_SDE_INTERFACE96#define FIX_APEI_GHES_SDEI_NORMAL __end_of_fixed_addresses97#define FIX_APEI_GHES_SDEI_CRITICAL __end_of_fixed_addresses98#endif99 100static ATOMIC_NOTIFIER_HEAD(ghes_report_chain);101 102static inline bool is_hest_type_generic_v2(struct ghes *ghes)103{104 return ghes->generic->header.type == ACPI_HEST_TYPE_GENERIC_ERROR_V2;105}106 107/*108 * A platform may describe one error source for the handling of synchronous109 * errors (e.g. MCE or SEA), or for handling asynchronous errors (e.g. SCI110 * or External Interrupt). On x86, the HEST notifications are always111 * asynchronous, so only SEA on ARM is delivered as a synchronous112 * notification.113 */114static inline bool is_hest_sync_notify(struct ghes *ghes)115{116 u8 notify_type = ghes->generic->notify.type;117 118 return notify_type == ACPI_HEST_NOTIFY_SEA;119}120 121/*122 * This driver isn't really modular, however for the time being,123 * continuing to use module_param is the easiest way to remain124 * compatible with existing boot arg use cases.125 */126bool ghes_disable;127module_param_named(disable, ghes_disable, bool, 0);128 129/*130 * "ghes.edac_force_enable" forcibly enables ghes_edac and skips the platform131 * check.132 */133static bool ghes_edac_force_enable;134module_param_named(edac_force_enable, ghes_edac_force_enable, bool, 0);135 136/*137 * All error sources notified with HED (Hardware Error Device) share a138 * single notifier callback, so they need to be linked and checked one139 * by one. This holds true for NMI too.140 *141 * RCU is used for these lists, so ghes_list_mutex is only used for142 * list changing, not for traversing.143 */144static LIST_HEAD(ghes_hed);145static DEFINE_MUTEX(ghes_list_mutex);146 147/*148 * A list of GHES devices which are given to the corresponding EDAC driver149 * ghes_edac for further use.150 */151static LIST_HEAD(ghes_devs);152static DEFINE_MUTEX(ghes_devs_mutex);153 154/*155 * Because the memory area used to transfer hardware error information156 * from BIOS to Linux can be determined only in NMI, IRQ or timer157 * handler, but general ioremap can not be used in atomic context, so158 * the fixmap is used instead.159 *160 * This spinlock is used to prevent the fixmap entry from being used161 * simultaneously.162 */163static DEFINE_SPINLOCK(ghes_notify_lock_irq);164 165struct ghes_vendor_record_entry {166 struct work_struct work;167 int error_severity;168 char vendor_record[];169};170 171static struct gen_pool *ghes_estatus_pool;172 173static struct ghes_estatus_cache __rcu *ghes_estatus_caches[GHES_ESTATUS_CACHES_SIZE];174static atomic_t ghes_estatus_cache_alloced;175 176static int ghes_panic_timeout __read_mostly = 30;177 178static void __iomem *ghes_map(u64 pfn, enum fixed_addresses fixmap_idx)179{180 phys_addr_t paddr;181 pgprot_t prot;182 183 paddr = PFN_PHYS(pfn);184 prot = arch_apei_get_mem_attribute(paddr);185 __set_fixmap(fixmap_idx, paddr, prot);186 187 return (void __iomem *) __fix_to_virt(fixmap_idx);188}189 190static void ghes_unmap(void __iomem *vaddr, enum fixed_addresses fixmap_idx)191{192 int _idx = virt_to_fix((unsigned long)vaddr);193 194 WARN_ON_ONCE(fixmap_idx != _idx);195 clear_fixmap(fixmap_idx);196}197 198int ghes_estatus_pool_init(unsigned int num_ghes)199{200 unsigned long addr, len;201 int rc;202 203 ghes_estatus_pool = gen_pool_create(GHES_ESTATUS_POOL_MIN_ALLOC_ORDER, -1);204 if (!ghes_estatus_pool)205 return -ENOMEM;206 207 len = GHES_ESTATUS_CACHE_AVG_SIZE * GHES_ESTATUS_CACHE_ALLOCED_MAX;208 len += (num_ghes * GHES_ESOURCE_PREALLOC_MAX_SIZE);209 210 addr = (unsigned long)vmalloc(PAGE_ALIGN(len));211 if (!addr)212 goto err_pool_alloc;213 214 rc = gen_pool_add(ghes_estatus_pool, addr, PAGE_ALIGN(len), -1);215 if (rc)216 goto err_pool_add;217 218 return 0;219 220err_pool_add:221 vfree((void *)addr);222 223err_pool_alloc:224 gen_pool_destroy(ghes_estatus_pool);225 226 return -ENOMEM;227}228 229/**230 * ghes_estatus_pool_region_free - free previously allocated memory231 * from the ghes_estatus_pool.232 * @addr: address of memory to free.233 * @size: size of memory to free.234 *235 * Returns none.236 */237void ghes_estatus_pool_region_free(unsigned long addr, u32 size)238{239 gen_pool_free(ghes_estatus_pool, addr, size);240}241EXPORT_SYMBOL_GPL(ghes_estatus_pool_region_free);242 243static int map_gen_v2(struct ghes *ghes)244{245 return apei_map_generic_address(&ghes->generic_v2->read_ack_register);246}247 248static void unmap_gen_v2(struct ghes *ghes)249{250 apei_unmap_generic_address(&ghes->generic_v2->read_ack_register);251}252 253static void ghes_ack_error(struct acpi_hest_generic_v2 *gv2)254{255 int rc;256 u64 val = 0;257 258 rc = apei_read(&val, &gv2->read_ack_register);259 if (rc)260 return;261 262 val &= gv2->read_ack_preserve << gv2->read_ack_register.bit_offset;263 val |= gv2->read_ack_write << gv2->read_ack_register.bit_offset;264 265 apei_write(val, &gv2->read_ack_register);266}267 268static struct ghes *ghes_new(struct acpi_hest_generic *generic)269{270 struct ghes *ghes;271 unsigned int error_block_length;272 int rc;273 274 ghes = kzalloc(sizeof(*ghes), GFP_KERNEL);275 if (!ghes)276 return ERR_PTR(-ENOMEM);277 278 ghes->generic = generic;279 if (is_hest_type_generic_v2(ghes)) {280 rc = map_gen_v2(ghes);281 if (rc)282 goto err_free;283 }284 285 rc = apei_map_generic_address(&generic->error_status_address);286 if (rc)287 goto err_unmap_read_ack_addr;288 error_block_length = generic->error_block_length;289 if (error_block_length > GHES_ESTATUS_MAX_SIZE) {290 pr_warn(FW_WARN GHES_PFX291 "Error status block length is too long: %u for "292 "generic hardware error source: %d.\n",293 error_block_length, generic->header.source_id);294 error_block_length = GHES_ESTATUS_MAX_SIZE;295 }296 ghes->estatus = kmalloc(error_block_length, GFP_KERNEL);297 if (!ghes->estatus) {298 rc = -ENOMEM;299 goto err_unmap_status_addr;300 }301 302 return ghes;303 304err_unmap_status_addr:305 apei_unmap_generic_address(&generic->error_status_address);306err_unmap_read_ack_addr:307 if (is_hest_type_generic_v2(ghes))308 unmap_gen_v2(ghes);309err_free:310 kfree(ghes);311 return ERR_PTR(rc);312}313 314static void ghes_fini(struct ghes *ghes)315{316 kfree(ghes->estatus);317 apei_unmap_generic_address(&ghes->generic->error_status_address);318 if (is_hest_type_generic_v2(ghes))319 unmap_gen_v2(ghes);320}321 322static inline int ghes_severity(int severity)323{324 switch (severity) {325 case CPER_SEV_INFORMATIONAL:326 return GHES_SEV_NO;327 case CPER_SEV_CORRECTED:328 return GHES_SEV_CORRECTED;329 case CPER_SEV_RECOVERABLE:330 return GHES_SEV_RECOVERABLE;331 case CPER_SEV_FATAL:332 return GHES_SEV_PANIC;333 default:334 /* Unknown, go panic */335 return GHES_SEV_PANIC;336 }337}338 339static void ghes_copy_tofrom_phys(void *buffer, u64 paddr, u32 len,340 int from_phys,341 enum fixed_addresses fixmap_idx)342{343 void __iomem *vaddr;344 u64 offset;345 u32 trunk;346 347 while (len > 0) {348 offset = paddr - (paddr & PAGE_MASK);349 vaddr = ghes_map(PHYS_PFN(paddr), fixmap_idx);350 trunk = PAGE_SIZE - offset;351 trunk = min(trunk, len);352 if (from_phys)353 memcpy_fromio(buffer, vaddr + offset, trunk);354 else355 memcpy_toio(vaddr + offset, buffer, trunk);356 len -= trunk;357 paddr += trunk;358 buffer += trunk;359 ghes_unmap(vaddr, fixmap_idx);360 }361}362 363/* Check the top-level record header has an appropriate size. */364static int __ghes_check_estatus(struct ghes *ghes,365 struct acpi_hest_generic_status *estatus)366{367 u32 len = cper_estatus_len(estatus);368 369 if (len < sizeof(*estatus)) {370 pr_warn_ratelimited(FW_WARN GHES_PFX "Truncated error status block!\n");371 return -EIO;372 }373 374 if (len > ghes->generic->error_block_length) {375 pr_warn_ratelimited(FW_WARN GHES_PFX "Invalid error status block length!\n");376 return -EIO;377 }378 379 if (cper_estatus_check_header(estatus)) {380 pr_warn_ratelimited(FW_WARN GHES_PFX "Invalid CPER header!\n");381 return -EIO;382 }383 384 return 0;385}386 387/* Read the CPER block, returning its address, and header in estatus. */388static int __ghes_peek_estatus(struct ghes *ghes,389 struct acpi_hest_generic_status *estatus,390 u64 *buf_paddr, enum fixed_addresses fixmap_idx)391{392 struct acpi_hest_generic *g = ghes->generic;393 int rc;394 395 rc = apei_read(buf_paddr, &g->error_status_address);396 if (rc) {397 *buf_paddr = 0;398 pr_warn_ratelimited(FW_WARN GHES_PFX399"Failed to read error status block address for hardware error source: %d.\n",400 g->header.source_id);401 return -EIO;402 }403 if (!*buf_paddr)404 return -ENOENT;405 406 ghes_copy_tofrom_phys(estatus, *buf_paddr, sizeof(*estatus), 1,407 fixmap_idx);408 if (!estatus->block_status) {409 *buf_paddr = 0;410 return -ENOENT;411 }412 413 return 0;414}415 416static int __ghes_read_estatus(struct acpi_hest_generic_status *estatus,417 u64 buf_paddr, enum fixed_addresses fixmap_idx,418 size_t buf_len)419{420 ghes_copy_tofrom_phys(estatus, buf_paddr, buf_len, 1, fixmap_idx);421 if (cper_estatus_check(estatus)) {422 pr_warn_ratelimited(FW_WARN GHES_PFX423 "Failed to read error status block!\n");424 return -EIO;425 }426 427 return 0;428}429 430static int ghes_read_estatus(struct ghes *ghes,431 struct acpi_hest_generic_status *estatus,432 u64 *buf_paddr, enum fixed_addresses fixmap_idx)433{434 int rc;435 436 rc = __ghes_peek_estatus(ghes, estatus, buf_paddr, fixmap_idx);437 if (rc)438 return rc;439 440 rc = __ghes_check_estatus(ghes, estatus);441 if (rc)442 return rc;443 444 return __ghes_read_estatus(estatus, *buf_paddr, fixmap_idx,445 cper_estatus_len(estatus));446}447 448static void ghes_clear_estatus(struct ghes *ghes,449 struct acpi_hest_generic_status *estatus,450 u64 buf_paddr, enum fixed_addresses fixmap_idx)451{452 estatus->block_status = 0;453 454 if (!buf_paddr)455 return;456 457 ghes_copy_tofrom_phys(estatus, buf_paddr,458 sizeof(estatus->block_status), 0,459 fixmap_idx);460 461 /*462 * GHESv2 type HEST entries introduce support for error acknowledgment,463 * so only acknowledge the error if this support is present.464 */465 if (is_hest_type_generic_v2(ghes))466 ghes_ack_error(ghes->generic_v2);467}468 469/*470 * Called as task_work before returning to user-space.471 * Ensure any queued work has been done before we return to the context that472 * triggered the notification.473 */474static void ghes_kick_task_work(struct callback_head *head)475{476 struct acpi_hest_generic_status *estatus;477 struct ghes_estatus_node *estatus_node;478 u32 node_len;479 480 estatus_node = container_of(head, struct ghes_estatus_node, task_work);481 if (IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE))482 memory_failure_queue_kick(estatus_node->task_work_cpu);483 484 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);485 node_len = GHES_ESTATUS_NODE_LEN(cper_estatus_len(estatus));486 gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node, node_len);487}488 489static bool ghes_do_memory_failure(u64 physical_addr, int flags)490{491 unsigned long pfn;492 493 if (!IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE))494 return false;495 496 pfn = PHYS_PFN(physical_addr);497 if (!pfn_valid(pfn) && !arch_is_platform_page(physical_addr)) {498 pr_warn_ratelimited(FW_WARN GHES_PFX499 "Invalid address in generic error data: %#llx\n",500 physical_addr);501 return false;502 }503 504 memory_failure_queue(pfn, flags);505 return true;506}507 508static bool ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata,509 int sev, bool sync)510{511 int flags = -1;512 int sec_sev = ghes_severity(gdata->error_severity);513 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata);514 515 if (!(mem_err->validation_bits & CPER_MEM_VALID_PA))516 return false;517 518 /* iff following two events can be handled properly by now */519 if (sec_sev == GHES_SEV_CORRECTED &&520 (gdata->flags & CPER_SEC_ERROR_THRESHOLD_EXCEEDED))521 flags = MF_SOFT_OFFLINE;522 if (sev == GHES_SEV_RECOVERABLE && sec_sev == GHES_SEV_RECOVERABLE)523 flags = sync ? MF_ACTION_REQUIRED : 0;524 525 if (flags != -1)526 return ghes_do_memory_failure(mem_err->physical_addr, flags);527 528 return false;529}530 531static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata,532 int sev, bool sync)533{534 struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata);535 int flags = sync ? MF_ACTION_REQUIRED : 0;536 bool queued = false;537 int sec_sev, i;538 char *p;539 540 log_arm_hw_error(err);541 542 sec_sev = ghes_severity(gdata->error_severity);543 if (sev != GHES_SEV_RECOVERABLE || sec_sev != GHES_SEV_RECOVERABLE)544 return false;545 546 p = (char *)(err + 1);547 for (i = 0; i < err->err_info_num; i++) {548 struct cper_arm_err_info *err_info = (struct cper_arm_err_info *)p;549 bool is_cache = (err_info->type == CPER_ARM_CACHE_ERROR);550 bool has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR);551 const char *error_type = "unknown error";552 553 /*554 * The field (err_info->error_info & BIT(26)) is fixed to set to555 * 1 in some old firmware of HiSilicon Kunpeng920. We assume that556 * firmware won't mix corrected errors in an uncorrected section,557 * and don't filter out 'corrected' error here.558 */559 if (is_cache && has_pa) {560 queued = ghes_do_memory_failure(err_info->physical_fault_addr, flags);561 p += err_info->length;562 continue;563 }564 565 if (err_info->type < ARRAY_SIZE(cper_proc_error_type_strs))566 error_type = cper_proc_error_type_strs[err_info->type];567 568 pr_warn_ratelimited(FW_WARN GHES_PFX569 "Unhandled processor error type: %s\n",570 error_type);571 p += err_info->length;572 }573 574 return queued;575}576 577/*578 * PCIe AER errors need to be sent to the AER driver for reporting and579 * recovery. The GHES severities map to the following AER severities and580 * require the following handling:581 *582 * GHES_SEV_CORRECTABLE -> AER_CORRECTABLE583 * These need to be reported by the AER driver but no recovery is584 * necessary.585 * GHES_SEV_RECOVERABLE -> AER_NONFATAL586 * GHES_SEV_RECOVERABLE && CPER_SEC_RESET -> AER_FATAL587 * These both need to be reported and recovered from by the AER driver.588 * GHES_SEV_PANIC does not make it to this handling since the kernel must589 * panic.590 */591static void ghes_handle_aer(struct acpi_hest_generic_data *gdata)592{593#ifdef CONFIG_ACPI_APEI_PCIEAER594 struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata);595 596 if (pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID &&597 pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) {598 unsigned int devfn;599 int aer_severity;600 u8 *aer_info;601 602 devfn = PCI_DEVFN(pcie_err->device_id.device,603 pcie_err->device_id.function);604 aer_severity = cper_severity_to_aer(gdata->error_severity);605 606 /*607 * If firmware reset the component to contain608 * the error, we must reinitialize it before609 * use, so treat it as a fatal AER error.610 */611 if (gdata->flags & CPER_SEC_RESET)612 aer_severity = AER_FATAL;613 614 aer_info = (void *)gen_pool_alloc(ghes_estatus_pool,615 sizeof(struct aer_capability_regs));616 if (!aer_info)617 return;618 memcpy(aer_info, pcie_err->aer_info, sizeof(struct aer_capability_regs));619 620 aer_recover_queue(pcie_err->device_id.segment,621 pcie_err->device_id.bus,622 devfn, aer_severity,623 (struct aer_capability_regs *)624 aer_info);625 }626#endif627}628 629static BLOCKING_NOTIFIER_HEAD(vendor_record_notify_list);630 631int ghes_register_vendor_record_notifier(struct notifier_block *nb)632{633 return blocking_notifier_chain_register(&vendor_record_notify_list, nb);634}635EXPORT_SYMBOL_GPL(ghes_register_vendor_record_notifier);636 637void ghes_unregister_vendor_record_notifier(struct notifier_block *nb)638{639 blocking_notifier_chain_unregister(&vendor_record_notify_list, nb);640}641EXPORT_SYMBOL_GPL(ghes_unregister_vendor_record_notifier);642 643static void ghes_vendor_record_work_func(struct work_struct *work)644{645 struct ghes_vendor_record_entry *entry;646 struct acpi_hest_generic_data *gdata;647 u32 len;648 649 entry = container_of(work, struct ghes_vendor_record_entry, work);650 gdata = GHES_GDATA_FROM_VENDOR_ENTRY(entry);651 652 blocking_notifier_call_chain(&vendor_record_notify_list,653 entry->error_severity, gdata);654 655 len = GHES_VENDOR_ENTRY_LEN(acpi_hest_get_record_size(gdata));656 gen_pool_free(ghes_estatus_pool, (unsigned long)entry, len);657}658 659static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata,660 int sev)661{662 struct acpi_hest_generic_data *copied_gdata;663 struct ghes_vendor_record_entry *entry;664 u32 len;665 666 len = GHES_VENDOR_ENTRY_LEN(acpi_hest_get_record_size(gdata));667 entry = (void *)gen_pool_alloc(ghes_estatus_pool, len);668 if (!entry)669 return;670 671 copied_gdata = GHES_GDATA_FROM_VENDOR_ENTRY(entry);672 memcpy(copied_gdata, gdata, acpi_hest_get_record_size(gdata));673 entry->error_severity = sev;674 675 INIT_WORK(&entry->work, ghes_vendor_record_work_func);676 schedule_work(&entry->work);677}678 679/* Room for 8 entries for each of the 4 event log queues */680#define CXL_CPER_FIFO_DEPTH 32681DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH);682 683/* Synchronize schedule_work() with cxl_cper_work changes */684static DEFINE_SPINLOCK(cxl_cper_work_lock);685struct work_struct *cxl_cper_work;686 687static void cxl_cper_post_event(enum cxl_event_type event_type,688 struct cxl_cper_event_rec *rec)689{690 struct cxl_cper_work_data wd;691 692 if (rec->hdr.length <= sizeof(rec->hdr) ||693 rec->hdr.length > sizeof(*rec)) {694 pr_err(FW_WARN "CXL CPER Invalid section length (%u)\n",695 rec->hdr.length);696 return;697 }698 699 if (!(rec->hdr.validation_bits & CPER_CXL_COMP_EVENT_LOG_VALID)) {700 pr_err(FW_WARN "CXL CPER invalid event\n");701 return;702 }703 704 guard(spinlock_irqsave)(&cxl_cper_work_lock);705 706 if (!cxl_cper_work)707 return;708 709 wd.event_type = event_type;710 memcpy(&wd.rec, rec, sizeof(wd.rec));711 712 if (!kfifo_put(&cxl_cper_fifo, wd)) {713 pr_err_ratelimited("CXL CPER kfifo overflow\n");714 return;715 }716 717 schedule_work(cxl_cper_work);718}719 720int cxl_cper_register_work(struct work_struct *work)721{722 if (cxl_cper_work)723 return -EINVAL;724 725 guard(spinlock)(&cxl_cper_work_lock);726 cxl_cper_work = work;727 return 0;728}729EXPORT_SYMBOL_NS_GPL(cxl_cper_register_work, CXL);730 731int cxl_cper_unregister_work(struct work_struct *work)732{733 if (cxl_cper_work != work)734 return -EINVAL;735 736 guard(spinlock)(&cxl_cper_work_lock);737 cxl_cper_work = NULL;738 return 0;739}740EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_work, CXL);741 742int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd)743{744 return kfifo_get(&cxl_cper_fifo, wd);745}746EXPORT_SYMBOL_NS_GPL(cxl_cper_kfifo_get, CXL);747 748static bool ghes_do_proc(struct ghes *ghes,749 const struct acpi_hest_generic_status *estatus)750{751 int sev, sec_sev;752 struct acpi_hest_generic_data *gdata;753 guid_t *sec_type;754 const guid_t *fru_id = &guid_null;755 char *fru_text = "";756 bool queued = false;757 bool sync = is_hest_sync_notify(ghes);758 759 sev = ghes_severity(estatus->error_severity);760 apei_estatus_for_each_section(estatus, gdata) {761 sec_type = (guid_t *)gdata->section_type;762 sec_sev = ghes_severity(gdata->error_severity);763 if (gdata->validation_bits & CPER_SEC_VALID_FRU_ID)764 fru_id = (guid_t *)gdata->fru_id;765 766 if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT)767 fru_text = gdata->fru_text;768 769 if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) {770 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata);771 772 atomic_notifier_call_chain(&ghes_report_chain, sev, mem_err);773 774 arch_apei_report_mem_error(sev, mem_err);775 queued = ghes_handle_memory_failure(gdata, sev, sync);776 }777 else if (guid_equal(sec_type, &CPER_SEC_PCIE)) {778 ghes_handle_aer(gdata);779 }780 else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) {781 queued = ghes_handle_arm_hw_error(gdata, sev, sync);782 } else if (guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID)) {783 struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata);784 785 cxl_cper_post_event(CXL_CPER_EVENT_GEN_MEDIA, rec);786 } else if (guid_equal(sec_type, &CPER_SEC_CXL_DRAM_GUID)) {787 struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata);788 789 cxl_cper_post_event(CXL_CPER_EVENT_DRAM, rec);790 } else if (guid_equal(sec_type, &CPER_SEC_CXL_MEM_MODULE_GUID)) {791 struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata);792 793 cxl_cper_post_event(CXL_CPER_EVENT_MEM_MODULE, rec);794 } else {795 void *err = acpi_hest_get_payload(gdata);796 797 ghes_defer_non_standard_event(gdata, sev);798 log_non_standard_event(sec_type, fru_id, fru_text,799 sec_sev, err,800 gdata->error_data_length);801 }802 }803 804 return queued;805}806 807static void __ghes_print_estatus(const char *pfx,808 const struct acpi_hest_generic *generic,809 const struct acpi_hest_generic_status *estatus)810{811 static atomic_t seqno;812 unsigned int curr_seqno;813 char pfx_seq[64];814 815 if (pfx == NULL) {816 if (ghes_severity(estatus->error_severity) <=817 GHES_SEV_CORRECTED)818 pfx = KERN_WARNING;819 else820 pfx = KERN_ERR;821 }822 curr_seqno = atomic_inc_return(&seqno);823 snprintf(pfx_seq, sizeof(pfx_seq), "%s{%u}" HW_ERR, pfx, curr_seqno);824 printk("%s""Hardware error from APEI Generic Hardware Error Source: %d\n",825 pfx_seq, generic->header.source_id);826 cper_estatus_print(pfx_seq, estatus);827}828 829static int ghes_print_estatus(const char *pfx,830 const struct acpi_hest_generic *generic,831 const struct acpi_hest_generic_status *estatus)832{833 /* Not more than 2 messages every 5 seconds */834 static DEFINE_RATELIMIT_STATE(ratelimit_corrected, 5*HZ, 2);835 static DEFINE_RATELIMIT_STATE(ratelimit_uncorrected, 5*HZ, 2);836 struct ratelimit_state *ratelimit;837 838 if (ghes_severity(estatus->error_severity) <= GHES_SEV_CORRECTED)839 ratelimit = &ratelimit_corrected;840 else841 ratelimit = &ratelimit_uncorrected;842 if (__ratelimit(ratelimit)) {843 __ghes_print_estatus(pfx, generic, estatus);844 return 1;845 }846 return 0;847}848 849/*850 * GHES error status reporting throttle, to report more kinds of851 * errors, instead of just most frequently occurred errors.852 */853static int ghes_estatus_cached(struct acpi_hest_generic_status *estatus)854{855 u32 len;856 int i, cached = 0;857 unsigned long long now;858 struct ghes_estatus_cache *cache;859 struct acpi_hest_generic_status *cache_estatus;860 861 len = cper_estatus_len(estatus);862 rcu_read_lock();863 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) {864 cache = rcu_dereference(ghes_estatus_caches[i]);865 if (cache == NULL)866 continue;867 if (len != cache->estatus_len)868 continue;869 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache);870 if (memcmp(estatus, cache_estatus, len))871 continue;872 atomic_inc(&cache->count);873 now = sched_clock();874 if (now - cache->time_in < GHES_ESTATUS_IN_CACHE_MAX_NSEC)875 cached = 1;876 break;877 }878 rcu_read_unlock();879 return cached;880}881 882static struct ghes_estatus_cache *ghes_estatus_cache_alloc(883 struct acpi_hest_generic *generic,884 struct acpi_hest_generic_status *estatus)885{886 int alloced;887 u32 len, cache_len;888 struct ghes_estatus_cache *cache;889 struct acpi_hest_generic_status *cache_estatus;890 891 alloced = atomic_add_return(1, &ghes_estatus_cache_alloced);892 if (alloced > GHES_ESTATUS_CACHE_ALLOCED_MAX) {893 atomic_dec(&ghes_estatus_cache_alloced);894 return NULL;895 }896 len = cper_estatus_len(estatus);897 cache_len = GHES_ESTATUS_CACHE_LEN(len);898 cache = (void *)gen_pool_alloc(ghes_estatus_pool, cache_len);899 if (!cache) {900 atomic_dec(&ghes_estatus_cache_alloced);901 return NULL;902 }903 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache);904 memcpy(cache_estatus, estatus, len);905 cache->estatus_len = len;906 atomic_set(&cache->count, 0);907 cache->generic = generic;908 cache->time_in = sched_clock();909 return cache;910}911 912static void ghes_estatus_cache_rcu_free(struct rcu_head *head)913{914 struct ghes_estatus_cache *cache;915 u32 len;916 917 cache = container_of(head, struct ghes_estatus_cache, rcu);918 len = cper_estatus_len(GHES_ESTATUS_FROM_CACHE(cache));919 len = GHES_ESTATUS_CACHE_LEN(len);920 gen_pool_free(ghes_estatus_pool, (unsigned long)cache, len);921 atomic_dec(&ghes_estatus_cache_alloced);922}923 924static void925ghes_estatus_cache_add(struct acpi_hest_generic *generic,926 struct acpi_hest_generic_status *estatus)927{928 unsigned long long now, duration, period, max_period = 0;929 struct ghes_estatus_cache *cache, *new_cache;930 struct ghes_estatus_cache __rcu *victim;931 int i, slot = -1, count;932 933 new_cache = ghes_estatus_cache_alloc(generic, estatus);934 if (!new_cache)935 return;936 937 rcu_read_lock();938 now = sched_clock();939 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) {940 cache = rcu_dereference(ghes_estatus_caches[i]);941 if (cache == NULL) {942 slot = i;943 break;944 }945 duration = now - cache->time_in;946 if (duration >= GHES_ESTATUS_IN_CACHE_MAX_NSEC) {947 slot = i;948 break;949 }950 count = atomic_read(&cache->count);951 period = duration;952 do_div(period, (count + 1));953 if (period > max_period) {954 max_period = period;955 slot = i;956 }957 }958 rcu_read_unlock();959 960 if (slot != -1) {961 /*962 * Use release semantics to ensure that ghes_estatus_cached()963 * running on another CPU will see the updated cache fields if964 * it can see the new value of the pointer.965 */966 victim = xchg_release(&ghes_estatus_caches[slot],967 RCU_INITIALIZER(new_cache));968 969 /*970 * At this point, victim may point to a cached item different971 * from the one based on which we selected the slot. Instead of972 * going to the loop again to pick another slot, let's just973 * drop the other item anyway: this may cause a false cache974 * miss later on, but that won't cause any problems.975 */976 if (victim)977 call_rcu(&unrcu_pointer(victim)->rcu,978 ghes_estatus_cache_rcu_free);979 }980}981 982static void __ghes_panic(struct ghes *ghes,983 struct acpi_hest_generic_status *estatus,984 u64 buf_paddr, enum fixed_addresses fixmap_idx)985{986 __ghes_print_estatus(KERN_EMERG, ghes->generic, estatus);987 988 ghes_clear_estatus(ghes, estatus, buf_paddr, fixmap_idx);989 990 /* reboot to log the error! */991 if (!panic_timeout)992 panic_timeout = ghes_panic_timeout;993 panic("Fatal hardware error!");994}995 996static int ghes_proc(struct ghes *ghes)997{998 struct acpi_hest_generic_status *estatus = ghes->estatus;999 u64 buf_paddr;1000 int rc;1001 1002 rc = ghes_read_estatus(ghes, estatus, &buf_paddr, FIX_APEI_GHES_IRQ);1003 if (rc)1004 goto out;1005 1006 if (ghes_severity(estatus->error_severity) >= GHES_SEV_PANIC)1007 __ghes_panic(ghes, estatus, buf_paddr, FIX_APEI_GHES_IRQ);1008 1009 if (!ghes_estatus_cached(estatus)) {1010 if (ghes_print_estatus(NULL, ghes->generic, estatus))1011 ghes_estatus_cache_add(ghes->generic, estatus);1012 }1013 ghes_do_proc(ghes, estatus);1014 1015out:1016 ghes_clear_estatus(ghes, estatus, buf_paddr, FIX_APEI_GHES_IRQ);1017 1018 return rc;1019}1020 1021static void ghes_add_timer(struct ghes *ghes)1022{1023 struct acpi_hest_generic *g = ghes->generic;1024 unsigned long expire;1025 1026 if (!g->notify.poll_interval) {1027 pr_warn(FW_WARN GHES_PFX "Poll interval is 0 for generic hardware error source: %d, disabled.\n",1028 g->header.source_id);1029 return;1030 }1031 expire = jiffies + msecs_to_jiffies(g->notify.poll_interval);1032 ghes->timer.expires = round_jiffies_relative(expire);1033 add_timer(&ghes->timer);1034}1035 1036static void ghes_poll_func(struct timer_list *t)1037{1038 struct ghes *ghes = from_timer(ghes, t, timer);1039 unsigned long flags;1040 1041 spin_lock_irqsave(&ghes_notify_lock_irq, flags);1042 ghes_proc(ghes);1043 spin_unlock_irqrestore(&ghes_notify_lock_irq, flags);1044 if (!(ghes->flags & GHES_EXITING))1045 ghes_add_timer(ghes);1046}1047 1048static irqreturn_t ghes_irq_func(int irq, void *data)1049{1050 struct ghes *ghes = data;1051 unsigned long flags;1052 int rc;1053 1054 spin_lock_irqsave(&ghes_notify_lock_irq, flags);1055 rc = ghes_proc(ghes);1056 spin_unlock_irqrestore(&ghes_notify_lock_irq, flags);1057 if (rc)1058 return IRQ_NONE;1059 1060 return IRQ_HANDLED;1061}1062 1063static int ghes_notify_hed(struct notifier_block *this, unsigned long event,1064 void *data)1065{1066 struct ghes *ghes;1067 unsigned long flags;1068 int ret = NOTIFY_DONE;1069 1070 spin_lock_irqsave(&ghes_notify_lock_irq, flags);1071 rcu_read_lock();1072 list_for_each_entry_rcu(ghes, &ghes_hed, list) {1073 if (!ghes_proc(ghes))1074 ret = NOTIFY_OK;1075 }1076 rcu_read_unlock();1077 spin_unlock_irqrestore(&ghes_notify_lock_irq, flags);1078 1079 return ret;1080}1081 1082static struct notifier_block ghes_notifier_hed = {1083 .notifier_call = ghes_notify_hed,1084};1085 1086/*1087 * Handlers for CPER records may not be NMI safe. For example,1088 * memory_failure_queue() takes spinlocks and calls schedule_work_on().1089 * In any NMI-like handler, memory from ghes_estatus_pool is used to save1090 * estatus, and added to the ghes_estatus_llist. irq_work_queue() causes1091 * ghes_proc_in_irq() to run in IRQ context where each estatus in1092 * ghes_estatus_llist is processed.1093 *1094 * Memory from the ghes_estatus_pool is also used with the ghes_estatus_cache1095 * to suppress frequent messages.1096 */1097static struct llist_head ghes_estatus_llist;1098static struct irq_work ghes_proc_irq_work;1099 1100static void ghes_proc_in_irq(struct irq_work *irq_work)1101{1102 struct llist_node *llnode, *next;1103 struct ghes_estatus_node *estatus_node;1104 struct acpi_hest_generic *generic;1105 struct acpi_hest_generic_status *estatus;1106 bool task_work_pending;1107 u32 len, node_len;1108 int ret;1109 1110 llnode = llist_del_all(&ghes_estatus_llist);1111 /*1112 * Because the time order of estatus in list is reversed,1113 * revert it back to proper order.1114 */1115 llnode = llist_reverse_order(llnode);1116 while (llnode) {1117 next = llnode->next;1118 estatus_node = llist_entry(llnode, struct ghes_estatus_node,1119 llnode);1120 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);1121 len = cper_estatus_len(estatus);1122 node_len = GHES_ESTATUS_NODE_LEN(len);1123 task_work_pending = ghes_do_proc(estatus_node->ghes, estatus);1124 if (!ghes_estatus_cached(estatus)) {1125 generic = estatus_node->generic;1126 if (ghes_print_estatus(NULL, generic, estatus))1127 ghes_estatus_cache_add(generic, estatus);1128 }1129 1130 if (task_work_pending && current->mm) {1131 estatus_node->task_work.func = ghes_kick_task_work;1132 estatus_node->task_work_cpu = smp_processor_id();1133 ret = task_work_add(current, &estatus_node->task_work,1134 TWA_RESUME);1135 if (ret)1136 estatus_node->task_work.func = NULL;1137 }1138 1139 if (!estatus_node->task_work.func)1140 gen_pool_free(ghes_estatus_pool,1141 (unsigned long)estatus_node, node_len);1142 1143 llnode = next;1144 }1145}1146 1147static void ghes_print_queued_estatus(void)1148{1149 struct llist_node *llnode;1150 struct ghes_estatus_node *estatus_node;1151 struct acpi_hest_generic *generic;1152 struct acpi_hest_generic_status *estatus;1153 1154 llnode = llist_del_all(&ghes_estatus_llist);1155 /*1156 * Because the time order of estatus in list is reversed,1157 * revert it back to proper order.1158 */1159 llnode = llist_reverse_order(llnode);1160 while (llnode) {1161 estatus_node = llist_entry(llnode, struct ghes_estatus_node,1162 llnode);1163 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);1164 generic = estatus_node->generic;1165 ghes_print_estatus(NULL, generic, estatus);1166 llnode = llnode->next;1167 }1168}1169 1170static int ghes_in_nmi_queue_one_entry(struct ghes *ghes,1171 enum fixed_addresses fixmap_idx)1172{1173 struct acpi_hest_generic_status *estatus, tmp_header;1174 struct ghes_estatus_node *estatus_node;1175 u32 len, node_len;1176 u64 buf_paddr;1177 int sev, rc;1178 1179 if (!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG))1180 return -EOPNOTSUPP;1181 1182 rc = __ghes_peek_estatus(ghes, &tmp_header, &buf_paddr, fixmap_idx);1183 if (rc) {1184 ghes_clear_estatus(ghes, &tmp_header, buf_paddr, fixmap_idx);1185 return rc;1186 }1187 1188 rc = __ghes_check_estatus(ghes, &tmp_header);1189 if (rc) {1190 ghes_clear_estatus(ghes, &tmp_header, buf_paddr, fixmap_idx);1191 return rc;1192 }1193 1194 len = cper_estatus_len(&tmp_header);1195 node_len = GHES_ESTATUS_NODE_LEN(len);1196 estatus_node = (void *)gen_pool_alloc(ghes_estatus_pool, node_len);1197 if (!estatus_node)1198 return -ENOMEM;1199 1200 estatus_node->ghes = ghes;1201 estatus_node->generic = ghes->generic;1202 estatus_node->task_work.func = NULL;1203 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);1204 1205 if (__ghes_read_estatus(estatus, buf_paddr, fixmap_idx, len)) {1206 ghes_clear_estatus(ghes, estatus, buf_paddr, fixmap_idx);1207 rc = -ENOENT;1208 goto no_work;1209 }1210 1211 sev = ghes_severity(estatus->error_severity);1212 if (sev >= GHES_SEV_PANIC) {1213 ghes_print_queued_estatus();1214 __ghes_panic(ghes, estatus, buf_paddr, fixmap_idx);1215 }1216 1217 ghes_clear_estatus(ghes, &tmp_header, buf_paddr, fixmap_idx);1218 1219 /* This error has been reported before, don't process it again. */1220 if (ghes_estatus_cached(estatus))1221 goto no_work;1222 1223 llist_add(&estatus_node->llnode, &ghes_estatus_llist);1224 1225 return rc;1226 1227no_work:1228 gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node,1229 node_len);1230 1231 return rc;1232}1233 1234static int ghes_in_nmi_spool_from_list(struct list_head *rcu_list,1235 enum fixed_addresses fixmap_idx)1236{1237 int ret = -ENOENT;1238 struct ghes *ghes;1239 1240 rcu_read_lock();1241 list_for_each_entry_rcu(ghes, rcu_list, list) {1242 if (!ghes_in_nmi_queue_one_entry(ghes, fixmap_idx))1243 ret = 0;1244 }1245 rcu_read_unlock();1246 1247 if (IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) && !ret)1248 irq_work_queue(&ghes_proc_irq_work);1249 1250 return ret;1251}1252 1253#ifdef CONFIG_ACPI_APEI_SEA1254static LIST_HEAD(ghes_sea);1255 1256/*1257 * Return 0 only if one of the SEA error sources successfully reported an error1258 * record sent from the firmware.1259 */1260int ghes_notify_sea(void)1261{1262 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_sea);1263 int rv;1264 1265 raw_spin_lock(&ghes_notify_lock_sea);1266 rv = ghes_in_nmi_spool_from_list(&ghes_sea, FIX_APEI_GHES_SEA);1267 raw_spin_unlock(&ghes_notify_lock_sea);1268 1269 return rv;1270}1271 1272static void ghes_sea_add(struct ghes *ghes)1273{1274 mutex_lock(&ghes_list_mutex);1275 list_add_rcu(&ghes->list, &ghes_sea);1276 mutex_unlock(&ghes_list_mutex);1277}1278 1279static void ghes_sea_remove(struct ghes *ghes)1280{1281 mutex_lock(&ghes_list_mutex);1282 list_del_rcu(&ghes->list);1283 mutex_unlock(&ghes_list_mutex);1284 synchronize_rcu();1285}1286#else /* CONFIG_ACPI_APEI_SEA */1287static inline void ghes_sea_add(struct ghes *ghes) { }1288static inline void ghes_sea_remove(struct ghes *ghes) { }1289#endif /* CONFIG_ACPI_APEI_SEA */1290 1291#ifdef CONFIG_HAVE_ACPI_APEI_NMI1292/*1293 * NMI may be triggered on any CPU, so ghes_in_nmi is used for1294 * having only one concurrent reader.1295 */1296static atomic_t ghes_in_nmi = ATOMIC_INIT(0);1297 1298static LIST_HEAD(ghes_nmi);1299 1300static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs)1301{1302 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_nmi);1303 int ret = NMI_DONE;1304 1305 if (!atomic_add_unless(&ghes_in_nmi, 1, 1))1306 return ret;1307 1308 raw_spin_lock(&ghes_notify_lock_nmi);1309 if (!ghes_in_nmi_spool_from_list(&ghes_nmi, FIX_APEI_GHES_NMI))1310 ret = NMI_HANDLED;1311 raw_spin_unlock(&ghes_notify_lock_nmi);1312 1313 atomic_dec(&ghes_in_nmi);1314 return ret;1315}1316 1317static void ghes_nmi_add(struct ghes *ghes)1318{1319 mutex_lock(&ghes_list_mutex);1320 if (list_empty(&ghes_nmi))1321 register_nmi_handler(NMI_LOCAL, ghes_notify_nmi, 0, "ghes");1322 list_add_rcu(&ghes->list, &ghes_nmi);1323 mutex_unlock(&ghes_list_mutex);1324}1325 1326static void ghes_nmi_remove(struct ghes *ghes)1327{1328 mutex_lock(&ghes_list_mutex);1329 list_del_rcu(&ghes->list);1330 if (list_empty(&ghes_nmi))1331 unregister_nmi_handler(NMI_LOCAL, "ghes");1332 mutex_unlock(&ghes_list_mutex);1333 /*1334 * To synchronize with NMI handler, ghes can only be1335 * freed after NMI handler finishes.1336 */1337 synchronize_rcu();1338}1339#else /* CONFIG_HAVE_ACPI_APEI_NMI */1340static inline void ghes_nmi_add(struct ghes *ghes) { }1341static inline void ghes_nmi_remove(struct ghes *ghes) { }1342#endif /* CONFIG_HAVE_ACPI_APEI_NMI */1343 1344static void ghes_nmi_init_cxt(void)1345{1346 init_irq_work(&ghes_proc_irq_work, ghes_proc_in_irq);1347}1348 1349static int __ghes_sdei_callback(struct ghes *ghes,1350 enum fixed_addresses fixmap_idx)1351{1352 if (!ghes_in_nmi_queue_one_entry(ghes, fixmap_idx)) {1353 irq_work_queue(&ghes_proc_irq_work);1354 1355 return 0;1356 }1357 1358 return -ENOENT;1359}1360 1361static int ghes_sdei_normal_callback(u32 event_num, struct pt_regs *regs,1362 void *arg)1363{1364 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_sdei_normal);1365 struct ghes *ghes = arg;1366 int err;1367 1368 raw_spin_lock(&ghes_notify_lock_sdei_normal);1369 err = __ghes_sdei_callback(ghes, FIX_APEI_GHES_SDEI_NORMAL);1370 raw_spin_unlock(&ghes_notify_lock_sdei_normal);1371 1372 return err;1373}1374 1375static int ghes_sdei_critical_callback(u32 event_num, struct pt_regs *regs,1376 void *arg)1377{1378 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_sdei_critical);1379 struct ghes *ghes = arg;1380 int err;1381 1382 raw_spin_lock(&ghes_notify_lock_sdei_critical);1383 err = __ghes_sdei_callback(ghes, FIX_APEI_GHES_SDEI_CRITICAL);1384 raw_spin_unlock(&ghes_notify_lock_sdei_critical);1385 1386 return err;1387}1388 1389static int apei_sdei_register_ghes(struct ghes *ghes)1390{1391 if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))1392 return -EOPNOTSUPP;1393 1394 return sdei_register_ghes(ghes, ghes_sdei_normal_callback,1395 ghes_sdei_critical_callback);1396}1397 1398static int apei_sdei_unregister_ghes(struct ghes *ghes)1399{1400 if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))1401 return -EOPNOTSUPP;1402 1403 return sdei_unregister_ghes(ghes);1404}1405 1406static int ghes_probe(struct platform_device *ghes_dev)1407{1408 struct acpi_hest_generic *generic;1409 struct ghes *ghes = NULL;1410 unsigned long flags;1411 1412 int rc = -EINVAL;1413 1414 generic = *(struct acpi_hest_generic **)ghes_dev->dev.platform_data;1415 if (!generic->enabled)1416 return -ENODEV;1417 1418 switch (generic->notify.type) {1419 case ACPI_HEST_NOTIFY_POLLED:1420 case ACPI_HEST_NOTIFY_EXTERNAL:1421 case ACPI_HEST_NOTIFY_SCI:1422 case ACPI_HEST_NOTIFY_GSIV:1423 case ACPI_HEST_NOTIFY_GPIO:1424 break;1425 1426 case ACPI_HEST_NOTIFY_SEA:1427 if (!IS_ENABLED(CONFIG_ACPI_APEI_SEA)) {1428 pr_warn(GHES_PFX "Generic hardware error source: %d notified via SEA is not supported\n",1429 generic->header.source_id);1430 rc = -ENOTSUPP;1431 goto err;1432 }1433 break;1434 case ACPI_HEST_NOTIFY_NMI:1435 if (!IS_ENABLED(CONFIG_HAVE_ACPI_APEI_NMI)) {1436 pr_warn(GHES_PFX "Generic hardware error source: %d notified via NMI interrupt is not supported!\n",1437 generic->header.source_id);1438 goto err;1439 }1440 break;1441 case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED:1442 if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE)) {1443 pr_warn(GHES_PFX "Generic hardware error source: %d notified via SDE Interface is not supported!\n",1444 generic->header.source_id);1445 goto err;1446 }1447 break;1448 case ACPI_HEST_NOTIFY_LOCAL:1449 pr_warn(GHES_PFX "Generic hardware error source: %d notified via local interrupt is not supported!\n",1450 generic->header.source_id);1451 goto err;1452 default:1453 pr_warn(FW_WARN GHES_PFX "Unknown notification type: %u for generic hardware error source: %d\n",1454 generic->notify.type, generic->header.source_id);1455 goto err;1456 }1457 1458 rc = -EIO;1459 if (generic->error_block_length <1460 sizeof(struct acpi_hest_generic_status)) {1461 pr_warn(FW_BUG GHES_PFX "Invalid error block length: %u for generic hardware error source: %d\n",1462 generic->error_block_length, generic->header.source_id);1463 goto err;1464 }1465 ghes = ghes_new(generic);1466 if (IS_ERR(ghes)) {1467 rc = PTR_ERR(ghes);1468 ghes = NULL;1469 goto err;1470 }1471 1472 switch (generic->notify.type) {1473 case ACPI_HEST_NOTIFY_POLLED:1474 timer_setup(&ghes->timer, ghes_poll_func, 0);1475 ghes_add_timer(ghes);1476 break;1477 case ACPI_HEST_NOTIFY_EXTERNAL:1478 /* External interrupt vector is GSI */1479 rc = acpi_gsi_to_irq(generic->notify.vector, &ghes->irq);1480 if (rc) {1481 pr_err(GHES_PFX "Failed to map GSI to IRQ for generic hardware error source: %d\n",1482 generic->header.source_id);1483 goto err;1484 }1485 rc = request_irq(ghes->irq, ghes_irq_func, IRQF_SHARED,1486 "GHES IRQ", ghes);1487 if (rc) {1488 pr_err(GHES_PFX "Failed to register IRQ for generic hardware error source: %d\n",1489 generic->header.source_id);1490 goto err;1491 }1492 break;1493 1494 case ACPI_HEST_NOTIFY_SCI:1495 case ACPI_HEST_NOTIFY_GSIV:1496 case ACPI_HEST_NOTIFY_GPIO:1497 mutex_lock(&ghes_list_mutex);1498 if (list_empty(&ghes_hed))1499 register_acpi_hed_notifier(&ghes_notifier_hed);1500 list_add_rcu(&ghes->list, &ghes_hed);1501 mutex_unlock(&ghes_list_mutex);1502 break;1503 1504 case ACPI_HEST_NOTIFY_SEA:1505 ghes_sea_add(ghes);1506 break;1507 case ACPI_HEST_NOTIFY_NMI:1508 ghes_nmi_add(ghes);1509 break;1510 case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED:1511 rc = apei_sdei_register_ghes(ghes);1512 if (rc)1513 goto err;1514 break;1515 default:1516 BUG();1517 }1518 1519 platform_set_drvdata(ghes_dev, ghes);1520 1521 ghes->dev = &ghes_dev->dev;1522 1523 mutex_lock(&ghes_devs_mutex);1524 list_add_tail(&ghes->elist, &ghes_devs);1525 mutex_unlock(&ghes_devs_mutex);1526 1527 /* Handle any pending errors right away */1528 spin_lock_irqsave(&ghes_notify_lock_irq, flags);1529 ghes_proc(ghes);1530 spin_unlock_irqrestore(&ghes_notify_lock_irq, flags);1531 1532 return 0;1533 1534err:1535 if (ghes) {1536 ghes_fini(ghes);1537 kfree(ghes);1538 }1539 return rc;1540}1541 1542static void ghes_remove(struct platform_device *ghes_dev)1543{1544 int rc;1545 struct ghes *ghes;1546 struct acpi_hest_generic *generic;1547 1548 ghes = platform_get_drvdata(ghes_dev);1549 generic = ghes->generic;1550 1551 ghes->flags |= GHES_EXITING;1552 switch (generic->notify.type) {1553 case ACPI_HEST_NOTIFY_POLLED:1554 timer_shutdown_sync(&ghes->timer);1555 break;1556 case ACPI_HEST_NOTIFY_EXTERNAL:1557 free_irq(ghes->irq, ghes);1558 break;1559 1560 case ACPI_HEST_NOTIFY_SCI:1561 case ACPI_HEST_NOTIFY_GSIV:1562 case ACPI_HEST_NOTIFY_GPIO:1563 mutex_lock(&ghes_list_mutex);1564 list_del_rcu(&ghes->list);1565 if (list_empty(&ghes_hed))1566 unregister_acpi_hed_notifier(&ghes_notifier_hed);1567 mutex_unlock(&ghes_list_mutex);1568 synchronize_rcu();1569 break;1570 1571 case ACPI_HEST_NOTIFY_SEA:1572 ghes_sea_remove(ghes);1573 break;1574 case ACPI_HEST_NOTIFY_NMI:1575 ghes_nmi_remove(ghes);1576 break;1577 case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED:1578 rc = apei_sdei_unregister_ghes(ghes);1579 if (rc) {1580 /*1581 * Returning early results in a resource leak, but we're1582 * only here if stopping the hardware failed.1583 */1584 dev_err(&ghes_dev->dev, "Failed to unregister ghes (%pe)\n",1585 ERR_PTR(rc));1586 return;1587 }1588 break;1589 default:1590 BUG();1591 break;1592 }1593 1594 ghes_fini(ghes);1595 1596 mutex_lock(&ghes_devs_mutex);1597 list_del(&ghes->elist);1598 mutex_unlock(&ghes_devs_mutex);1599 1600 kfree(ghes);1601}1602 1603static struct platform_driver ghes_platform_driver = {1604 .driver = {1605 .name = "GHES",1606 },1607 .probe = ghes_probe,1608 .remove_new = ghes_remove,1609};1610 1611void __init acpi_ghes_init(void)1612{1613 int rc;1614 1615 sdei_init();1616 1617 if (acpi_disabled)1618 return;1619 1620 switch (hest_disable) {1621 case HEST_NOT_FOUND:1622 return;1623 case HEST_DISABLED:1624 pr_info(GHES_PFX "HEST is not enabled!\n");1625 return;1626 default:1627 break;1628 }1629 1630 if (ghes_disable) {1631 pr_info(GHES_PFX "GHES is not enabled!\n");1632 return;1633 }1634 1635 ghes_nmi_init_cxt();1636 1637 rc = platform_driver_register(&ghes_platform_driver);1638 if (rc)1639 return;1640 1641 rc = apei_osc_setup();1642 if (rc == 0 && osc_sb_apei_support_acked)1643 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit and WHEA _OSC.\n");1644 else if (rc == 0 && !osc_sb_apei_support_acked)1645 pr_info(GHES_PFX "APEI firmware first mode is enabled by WHEA _OSC.\n");1646 else if (rc && osc_sb_apei_support_acked)1647 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit.\n");1648 else1649 pr_info(GHES_PFX "Failed to enable APEI firmware first mode.\n");1650}1651 1652/*1653 * Known x86 systems that prefer GHES error reporting:1654 */1655static struct acpi_platform_list plat_list[] = {1656 {"HPE ", "Server ", 0, ACPI_SIG_FADT, all_versions},1657 { } /* End */1658};1659 1660struct list_head *ghes_get_devices(void)1661{1662 int idx = -1;1663 1664 if (IS_ENABLED(CONFIG_X86)) {1665 idx = acpi_match_platform_list(plat_list);1666 if (idx < 0) {1667 if (!ghes_edac_force_enable)1668 return NULL;1669 1670 pr_warn_once("Force-loading ghes_edac on an unsupported platform. You're on your own!\n");1671 }1672 } else if (list_empty(&ghes_devs)) {1673 return NULL;1674 }1675 1676 return &ghes_devs;1677}1678EXPORT_SYMBOL_GPL(ghes_get_devices);1679 1680void ghes_register_report_chain(struct notifier_block *nb)1681{1682 atomic_notifier_chain_register(&ghes_report_chain, nb);1683}1684EXPORT_SYMBOL_GPL(ghes_register_report_chain);1685 1686void ghes_unregister_report_chain(struct notifier_block *nb)1687{1688 atomic_notifier_chain_unregister(&ghes_report_chain, nb);1689}1690EXPORT_SYMBOL_GPL(ghes_unregister_report_chain);1691