1596 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Implement the AER root port service driver. The driver registers an IRQ4 * handler. When a root port triggers an AER interrupt, the IRQ handler5 * collects root port status and schedules work.6 *7 * Copyright (C) 2006 Intel Corp.8 * Tom Long Nguyen (tom.l.nguyen@intel.com)9 * Zhang Yanmin (yanmin.zhang@intel.com)10 *11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P.12 * Andrew Patterson <andrew.patterson@hp.com>13 */14 15#define pr_fmt(fmt) "AER: " fmt16#define dev_fmt pr_fmt17 18#include <linux/bitops.h>19#include <linux/cper.h>20#include <linux/pci.h>21#include <linux/pci-acpi.h>22#include <linux/sched.h>23#include <linux/kernel.h>24#include <linux/errno.h>25#include <linux/pm.h>26#include <linux/init.h>27#include <linux/interrupt.h>28#include <linux/delay.h>29#include <linux/kfifo.h>30#include <linux/slab.h>31#include <acpi/apei.h>32#include <acpi/ghes.h>33#include <ras/ras_event.h>34 35#include "../pci.h"36#include "portdrv.h"37 38#define AER_ERROR_SOURCES_MAX 12839 40#define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */41#define AER_MAX_TYPEOF_UNCOR_ERRS 27 /* as per PCI_ERR_UNCOR_STATUS*/42 43struct aer_err_source {44 u32 status; /* PCI_ERR_ROOT_STATUS */45 u32 id; /* PCI_ERR_ROOT_ERR_SRC */46};47 48struct aer_rpc {49 struct pci_dev *rpd; /* Root Port device */50 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX);51};52 53/* AER stats for the device */54struct aer_stats {55 56 /*57 * Fields for all AER capable devices. They indicate the errors58 * "as seen by this device". Note that this may mean that if an59 * end point is causing problems, the AER counters may increment60 * at its link partner (e.g. root port) because the errors will be61 * "seen" by the link partner and not the problematic end point62 * itself (which may report all counters as 0 as it never saw any63 * problems).64 */65 /* Counters for different type of correctable errors */66 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS];67 /* Counters for different type of fatal uncorrectable errors */68 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];69 /* Counters for different type of nonfatal uncorrectable errors */70 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];71 /* Total number of ERR_COR sent by this device */72 u64 dev_total_cor_errs;73 /* Total number of ERR_FATAL sent by this device */74 u64 dev_total_fatal_errs;75 /* Total number of ERR_NONFATAL sent by this device */76 u64 dev_total_nonfatal_errs;77 78 /*79 * Fields for Root ports & root complex event collectors only, these80 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL81 * messages received by the root port / event collector, INCLUDING the82 * ones that are generated internally (by the rootport itself)83 */84 u64 rootport_total_cor_errs;85 u64 rootport_total_fatal_errs;86 u64 rootport_total_nonfatal_errs;87};88 89#define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \90 PCI_ERR_UNC_ECRC| \91 PCI_ERR_UNC_UNSUP| \92 PCI_ERR_UNC_COMP_ABORT| \93 PCI_ERR_UNC_UNX_COMP| \94 PCI_ERR_UNC_MALF_TLP)95 96#define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \97 PCI_EXP_RTCTL_SENFEE| \98 PCI_EXP_RTCTL_SEFEE)99#define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \100 PCI_ERR_ROOT_CMD_NONFATAL_EN| \101 PCI_ERR_ROOT_CMD_FATAL_EN)102#define ERR_COR_ID(d) (d & 0xffff)103#define ERR_UNCOR_ID(d) (d >> 16)104 105#define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \106 PCI_ERR_ROOT_COR_RCV | \107 PCI_ERR_ROOT_MULTI_COR_RCV | \108 PCI_ERR_ROOT_MULTI_UNCOR_RCV)109 110static int pcie_aer_disable;111static pci_ers_result_t aer_root_reset(struct pci_dev *dev);112 113void pci_no_aer(void)114{115 pcie_aer_disable = 1;116}117 118bool pci_aer_available(void)119{120 return !pcie_aer_disable && pci_msi_enabled();121}122 123#ifdef CONFIG_PCIE_ECRC124 125#define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */126#define ECRC_POLICY_OFF 1 /* ECRC off for performance */127#define ECRC_POLICY_ON 2 /* ECRC on for data integrity */128 129static int ecrc_policy = ECRC_POLICY_DEFAULT;130 131static const char * const ecrc_policy_str[] = {132 [ECRC_POLICY_DEFAULT] = "bios",133 [ECRC_POLICY_OFF] = "off",134 [ECRC_POLICY_ON] = "on"135};136 137/**138 * enable_ecrc_checking - enable PCIe ECRC checking for a device139 * @dev: the PCI device140 *141 * Returns 0 on success, or negative on failure.142 */143static int enable_ecrc_checking(struct pci_dev *dev)144{145 int aer = dev->aer_cap;146 u32 reg32;147 148 if (!aer)149 return -ENODEV;150 151 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32);152 if (reg32 & PCI_ERR_CAP_ECRC_GENC)153 reg32 |= PCI_ERR_CAP_ECRC_GENE;154 if (reg32 & PCI_ERR_CAP_ECRC_CHKC)155 reg32 |= PCI_ERR_CAP_ECRC_CHKE;156 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);157 158 return 0;159}160 161/**162 * disable_ecrc_checking - disables PCIe ECRC checking for a device163 * @dev: the PCI device164 *165 * Returns 0 on success, or negative on failure.166 */167static int disable_ecrc_checking(struct pci_dev *dev)168{169 int aer = dev->aer_cap;170 u32 reg32;171 172 if (!aer)173 return -ENODEV;174 175 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32);176 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);177 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);178 179 return 0;180}181 182/**183 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based on global policy184 * @dev: the PCI device185 */186void pcie_set_ecrc_checking(struct pci_dev *dev)187{188 if (!pcie_aer_is_native(dev))189 return;190 191 switch (ecrc_policy) {192 case ECRC_POLICY_DEFAULT:193 return;194 case ECRC_POLICY_OFF:195 disable_ecrc_checking(dev);196 break;197 case ECRC_POLICY_ON:198 enable_ecrc_checking(dev);199 break;200 default:201 return;202 }203}204 205/**206 * pcie_ecrc_get_policy - parse kernel command-line ecrc option207 * @str: ECRC policy from kernel command line to use208 */209void pcie_ecrc_get_policy(char *str)210{211 int i;212 213 i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str);214 if (i < 0)215 return;216 217 ecrc_policy = i;218}219#endif /* CONFIG_PCIE_ECRC */220 221#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \222 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE)223 224int pcie_aer_is_native(struct pci_dev *dev)225{226 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);227 228 if (!dev->aer_cap)229 return 0;230 231 return pcie_ports_native || host->native_aer;232}233EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, CXL);234 235static int pci_enable_pcie_error_reporting(struct pci_dev *dev)236{237 int rc;238 239 if (!pcie_aer_is_native(dev))240 return -EIO;241 242 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);243 return pcibios_err_to_errno(rc);244}245 246int pci_aer_clear_nonfatal_status(struct pci_dev *dev)247{248 int aer = dev->aer_cap;249 u32 status, sev;250 251 if (!pcie_aer_is_native(dev))252 return -EIO;253 254 /* Clear status bits for ERR_NONFATAL errors only */255 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);256 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);257 status &= ~sev;258 if (status)259 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);260 261 return 0;262}263EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status);264 265void pci_aer_clear_fatal_status(struct pci_dev *dev)266{267 int aer = dev->aer_cap;268 u32 status, sev;269 270 if (!pcie_aer_is_native(dev))271 return;272 273 /* Clear status bits for ERR_FATAL errors only */274 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);275 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);276 status &= sev;277 if (status)278 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);279}280 281/**282 * pci_aer_raw_clear_status - Clear AER error registers.283 * @dev: the PCI device284 *285 * Clearing AER error status registers unconditionally, regardless of286 * whether they're owned by firmware or the OS.287 *288 * Returns 0 on success, or negative on failure.289 */290int pci_aer_raw_clear_status(struct pci_dev *dev)291{292 int aer = dev->aer_cap;293 u32 status;294 int port_type;295 296 if (!aer)297 return -EIO;298 299 port_type = pci_pcie_type(dev);300 if (port_type == PCI_EXP_TYPE_ROOT_PORT ||301 port_type == PCI_EXP_TYPE_RC_EC) {302 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status);303 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status);304 }305 306 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);307 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status);308 309 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);310 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);311 312 return 0;313}314 315int pci_aer_clear_status(struct pci_dev *dev)316{317 if (!pcie_aer_is_native(dev))318 return -EIO;319 320 return pci_aer_raw_clear_status(dev);321}322 323void pci_save_aer_state(struct pci_dev *dev)324{325 int aer = dev->aer_cap;326 struct pci_cap_saved_state *save_state;327 u32 *cap;328 329 if (!aer)330 return;331 332 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);333 if (!save_state)334 return;335 336 cap = &save_state->cap.data[0];337 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++);338 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++);339 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++);340 pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++);341 if (pcie_cap_has_rtctl(dev))342 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++);343}344 345void pci_restore_aer_state(struct pci_dev *dev)346{347 int aer = dev->aer_cap;348 struct pci_cap_saved_state *save_state;349 u32 *cap;350 351 if (!aer)352 return;353 354 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);355 if (!save_state)356 return;357 358 cap = &save_state->cap.data[0];359 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++);360 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++);361 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++);362 pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++);363 if (pcie_cap_has_rtctl(dev))364 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++);365}366 367void pci_aer_init(struct pci_dev *dev)368{369 int n;370 371 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);372 if (!dev->aer_cap)373 return;374 375 dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL);376 377 /*378 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER,379 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event380 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r5.0, sec381 * 7.8.4).382 */383 n = pcie_cap_has_rtctl(dev) ? 5 : 4;384 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n);385 386 pci_aer_clear_status(dev);387 388 if (pci_aer_available())389 pci_enable_pcie_error_reporting(dev);390 391 pcie_set_ecrc_checking(dev);392}393 394void pci_aer_exit(struct pci_dev *dev)395{396 kfree(dev->aer_stats);397 dev->aer_stats = NULL;398}399 400#define AER_AGENT_RECEIVER 0401#define AER_AGENT_REQUESTER 1402#define AER_AGENT_COMPLETER 2403#define AER_AGENT_TRANSMITTER 3404 405#define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \406 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP))407#define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \408 0 : PCI_ERR_UNC_COMP_ABORT)409#define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \410 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0)411 412#define AER_GET_AGENT(t, e) \413 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \414 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \415 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \416 AER_AGENT_RECEIVER)417 418#define AER_PHYSICAL_LAYER_ERROR 0419#define AER_DATA_LINK_LAYER_ERROR 1420#define AER_TRANSACTION_LAYER_ERROR 2421 422#define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \423 PCI_ERR_COR_RCVR : 0)424#define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \425 (PCI_ERR_COR_BAD_TLP| \426 PCI_ERR_COR_BAD_DLLP| \427 PCI_ERR_COR_REP_ROLL| \428 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP)429 430#define AER_GET_LAYER_ERROR(t, e) \431 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \432 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \433 AER_TRANSACTION_LAYER_ERROR)434 435/*436 * AER error strings437 */438static const char * const aer_error_severity_string[] = {439 "Uncorrectable (Non-Fatal)",440 "Uncorrectable (Fatal)",441 "Correctable"442};443 444static const char *aer_error_layer[] = {445 "Physical Layer",446 "Data Link Layer",447 "Transaction Layer"448};449 450static const char *aer_correctable_error_string[] = {451 "RxErr", /* Bit Position 0 */452 NULL,453 NULL,454 NULL,455 NULL,456 NULL,457 "BadTLP", /* Bit Position 6 */458 "BadDLLP", /* Bit Position 7 */459 "Rollover", /* Bit Position 8 */460 NULL,461 NULL,462 NULL,463 "Timeout", /* Bit Position 12 */464 "NonFatalErr", /* Bit Position 13 */465 "CorrIntErr", /* Bit Position 14 */466 "HeaderOF", /* Bit Position 15 */467 NULL, /* Bit Position 16 */468 NULL, /* Bit Position 17 */469 NULL, /* Bit Position 18 */470 NULL, /* Bit Position 19 */471 NULL, /* Bit Position 20 */472 NULL, /* Bit Position 21 */473 NULL, /* Bit Position 22 */474 NULL, /* Bit Position 23 */475 NULL, /* Bit Position 24 */476 NULL, /* Bit Position 25 */477 NULL, /* Bit Position 26 */478 NULL, /* Bit Position 27 */479 NULL, /* Bit Position 28 */480 NULL, /* Bit Position 29 */481 NULL, /* Bit Position 30 */482 NULL, /* Bit Position 31 */483};484 485static const char *aer_uncorrectable_error_string[] = {486 "Undefined", /* Bit Position 0 */487 NULL,488 NULL,489 NULL,490 "DLP", /* Bit Position 4 */491 "SDES", /* Bit Position 5 */492 NULL,493 NULL,494 NULL,495 NULL,496 NULL,497 NULL,498 "TLP", /* Bit Position 12 */499 "FCP", /* Bit Position 13 */500 "CmpltTO", /* Bit Position 14 */501 "CmpltAbrt", /* Bit Position 15 */502 "UnxCmplt", /* Bit Position 16 */503 "RxOF", /* Bit Position 17 */504 "MalfTLP", /* Bit Position 18 */505 "ECRC", /* Bit Position 19 */506 "UnsupReq", /* Bit Position 20 */507 "ACSViol", /* Bit Position 21 */508 "UncorrIntErr", /* Bit Position 22 */509 "BlockedTLP", /* Bit Position 23 */510 "AtomicOpBlocked", /* Bit Position 24 */511 "TLPBlockedErr", /* Bit Position 25 */512 "PoisonTLPBlocked", /* Bit Position 26 */513 NULL, /* Bit Position 27 */514 NULL, /* Bit Position 28 */515 NULL, /* Bit Position 29 */516 NULL, /* Bit Position 30 */517 NULL, /* Bit Position 31 */518};519 520static const char *aer_agent_string[] = {521 "Receiver ID",522 "Requester ID",523 "Completer ID",524 "Transmitter ID"525};526 527#define aer_stats_dev_attr(name, stats_array, strings_array, \528 total_string, total_field) \529 static ssize_t \530 name##_show(struct device *dev, struct device_attribute *attr, \531 char *buf) \532{ \533 unsigned int i; \534 struct pci_dev *pdev = to_pci_dev(dev); \535 u64 *stats = pdev->aer_stats->stats_array; \536 size_t len = 0; \537 \538 for (i = 0; i < ARRAY_SIZE(pdev->aer_stats->stats_array); i++) {\539 if (strings_array[i]) \540 len += sysfs_emit_at(buf, len, "%s %llu\n", \541 strings_array[i], \542 stats[i]); \543 else if (stats[i]) \544 len += sysfs_emit_at(buf, len, \545 #stats_array "_bit[%d] %llu\n",\546 i, stats[i]); \547 } \548 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \549 pdev->aer_stats->total_field); \550 return len; \551} \552static DEVICE_ATTR_RO(name)553 554aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs,555 aer_correctable_error_string, "ERR_COR",556 dev_total_cor_errs);557aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs,558 aer_uncorrectable_error_string, "ERR_FATAL",559 dev_total_fatal_errs);560aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs,561 aer_uncorrectable_error_string, "ERR_NONFATAL",562 dev_total_nonfatal_errs);563 564#define aer_stats_rootport_attr(name, field) \565 static ssize_t \566 name##_show(struct device *dev, struct device_attribute *attr, \567 char *buf) \568{ \569 struct pci_dev *pdev = to_pci_dev(dev); \570 return sysfs_emit(buf, "%llu\n", pdev->aer_stats->field); \571} \572static DEVICE_ATTR_RO(name)573 574aer_stats_rootport_attr(aer_rootport_total_err_cor,575 rootport_total_cor_errs);576aer_stats_rootport_attr(aer_rootport_total_err_fatal,577 rootport_total_fatal_errs);578aer_stats_rootport_attr(aer_rootport_total_err_nonfatal,579 rootport_total_nonfatal_errs);580 581static struct attribute *aer_stats_attrs[] __ro_after_init = {582 &dev_attr_aer_dev_correctable.attr,583 &dev_attr_aer_dev_fatal.attr,584 &dev_attr_aer_dev_nonfatal.attr,585 &dev_attr_aer_rootport_total_err_cor.attr,586 &dev_attr_aer_rootport_total_err_fatal.attr,587 &dev_attr_aer_rootport_total_err_nonfatal.attr,588 NULL589};590 591static umode_t aer_stats_attrs_are_visible(struct kobject *kobj,592 struct attribute *a, int n)593{594 struct device *dev = kobj_to_dev(kobj);595 struct pci_dev *pdev = to_pci_dev(dev);596 597 if (!pdev->aer_stats)598 return 0;599 600 if ((a == &dev_attr_aer_rootport_total_err_cor.attr ||601 a == &dev_attr_aer_rootport_total_err_fatal.attr ||602 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) &&603 ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) &&604 (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC)))605 return 0;606 607 return a->mode;608}609 610const struct attribute_group aer_stats_attr_group = {611 .attrs = aer_stats_attrs,612 .is_visible = aer_stats_attrs_are_visible,613};614 615static void pci_dev_aer_stats_incr(struct pci_dev *pdev,616 struct aer_err_info *info)617{618 unsigned long status = info->status & ~info->mask;619 int i, max = -1;620 u64 *counter = NULL;621 struct aer_stats *aer_stats = pdev->aer_stats;622 623 if (!aer_stats)624 return;625 626 switch (info->severity) {627 case AER_CORRECTABLE:628 aer_stats->dev_total_cor_errs++;629 counter = &aer_stats->dev_cor_errs[0];630 max = AER_MAX_TYPEOF_COR_ERRS;631 break;632 case AER_NONFATAL:633 aer_stats->dev_total_nonfatal_errs++;634 counter = &aer_stats->dev_nonfatal_errs[0];635 max = AER_MAX_TYPEOF_UNCOR_ERRS;636 break;637 case AER_FATAL:638 aer_stats->dev_total_fatal_errs++;639 counter = &aer_stats->dev_fatal_errs[0];640 max = AER_MAX_TYPEOF_UNCOR_ERRS;641 break;642 }643 644 for_each_set_bit(i, &status, max)645 counter[i]++;646}647 648static void pci_rootport_aer_stats_incr(struct pci_dev *pdev,649 struct aer_err_source *e_src)650{651 struct aer_stats *aer_stats = pdev->aer_stats;652 653 if (!aer_stats)654 return;655 656 if (e_src->status & PCI_ERR_ROOT_COR_RCV)657 aer_stats->rootport_total_cor_errs++;658 659 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {660 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)661 aer_stats->rootport_total_fatal_errs++;662 else663 aer_stats->rootport_total_nonfatal_errs++;664 }665}666 667static void __print_tlp_header(struct pci_dev *dev, struct pcie_tlp_log *t)668{669 pci_err(dev, " TLP Header: %08x %08x %08x %08x\n",670 t->dw[0], t->dw[1], t->dw[2], t->dw[3]);671}672 673static void __aer_print_error(struct pci_dev *dev,674 struct aer_err_info *info)675{676 const char **strings;677 unsigned long status = info->status & ~info->mask;678 const char *level, *errmsg;679 int i;680 681 if (info->severity == AER_CORRECTABLE) {682 strings = aer_correctable_error_string;683 level = KERN_WARNING;684 } else {685 strings = aer_uncorrectable_error_string;686 level = KERN_ERR;687 }688 689 for_each_set_bit(i, &status, 32) {690 errmsg = strings[i];691 if (!errmsg)692 errmsg = "Unknown Error Bit";693 694 pci_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg,695 info->first_error == i ? " (First)" : "");696 }697 pci_dev_aer_stats_incr(dev, info);698}699 700void aer_print_error(struct pci_dev *dev, struct aer_err_info *info)701{702 int layer, agent;703 int id = pci_dev_id(dev);704 const char *level;705 706 if (!info->status) {707 pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",708 aer_error_severity_string[info->severity]);709 goto out;710 }711 712 layer = AER_GET_LAYER_ERROR(info->severity, info->status);713 agent = AER_GET_AGENT(info->severity, info->status);714 715 level = (info->severity == AER_CORRECTABLE) ? KERN_WARNING : KERN_ERR;716 717 pci_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",718 aer_error_severity_string[info->severity],719 aer_error_layer[layer], aer_agent_string[agent]);720 721 pci_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n",722 dev->vendor, dev->device, info->status, info->mask);723 724 __aer_print_error(dev, info);725 726 if (info->tlp_header_valid)727 __print_tlp_header(dev, &info->tlp);728 729out:730 if (info->id && info->error_dev_num > 1 && info->id == id)731 pci_err(dev, " Error of this Agent is reported first\n");732 733 trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask),734 info->severity, info->tlp_header_valid, &info->tlp);735}736 737static void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info)738{739 u8 bus = info->id >> 8;740 u8 devfn = info->id & 0xff;741 742 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d\n",743 info->multi_error_valid ? "Multiple " : "",744 aer_error_severity_string[info->severity],745 pci_domain_nr(dev->bus), bus, PCI_SLOT(devfn),746 PCI_FUNC(devfn));747}748 749#ifdef CONFIG_ACPI_APEI_PCIEAER750int cper_severity_to_aer(int cper_severity)751{752 switch (cper_severity) {753 case CPER_SEV_RECOVERABLE:754 return AER_NONFATAL;755 case CPER_SEV_FATAL:756 return AER_FATAL;757 default:758 return AER_CORRECTABLE;759 }760}761EXPORT_SYMBOL_GPL(cper_severity_to_aer);762#endif763 764void pci_print_aer(struct pci_dev *dev, int aer_severity,765 struct aer_capability_regs *aer)766{767 int layer, agent, tlp_header_valid = 0;768 u32 status, mask;769 struct aer_err_info info;770 771 if (aer_severity == AER_CORRECTABLE) {772 status = aer->cor_status;773 mask = aer->cor_mask;774 } else {775 status = aer->uncor_status;776 mask = aer->uncor_mask;777 tlp_header_valid = status & AER_LOG_TLP_MASKS;778 }779 780 layer = AER_GET_LAYER_ERROR(aer_severity, status);781 agent = AER_GET_AGENT(aer_severity, status);782 783 memset(&info, 0, sizeof(info));784 info.severity = aer_severity;785 info.status = status;786 info.mask = mask;787 info.first_error = PCI_ERR_CAP_FEP(aer->cap_control);788 789 pci_err(dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", status, mask);790 __aer_print_error(dev, &info);791 pci_err(dev, "aer_layer=%s, aer_agent=%s\n",792 aer_error_layer[layer], aer_agent_string[agent]);793 794 if (aer_severity != AER_CORRECTABLE)795 pci_err(dev, "aer_uncor_severity: 0x%08x\n",796 aer->uncor_severity);797 798 if (tlp_header_valid)799 __print_tlp_header(dev, &aer->header_log);800 801 trace_aer_event(dev_name(&dev->dev), (status & ~mask),802 aer_severity, tlp_header_valid, &aer->header_log);803}804EXPORT_SYMBOL_NS_GPL(pci_print_aer, CXL);805 806/**807 * add_error_device - list device to be handled808 * @e_info: pointer to error info809 * @dev: pointer to pci_dev to be added810 */811static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)812{813 if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {814 e_info->dev[e_info->error_dev_num] = pci_dev_get(dev);815 e_info->error_dev_num++;816 return 0;817 }818 return -ENOSPC;819}820 821/**822 * is_error_source - check whether the device is source of reported error823 * @dev: pointer to pci_dev to be checked824 * @e_info: pointer to reported error info825 */826static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)827{828 int aer = dev->aer_cap;829 u32 status, mask;830 u16 reg16;831 832 /*833 * When bus id is equal to 0, it might be a bad id834 * reported by root port.835 */836 if ((PCI_BUS_NUM(e_info->id) != 0) &&837 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {838 /* Device ID match? */839 if (e_info->id == pci_dev_id(dev))840 return true;841 842 /* Continue id comparing if there is no multiple error */843 if (!e_info->multi_error_valid)844 return false;845 }846 847 /*848 * When either849 * 1) bus id is equal to 0. Some ports might lose the bus850 * id of error source id;851 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set852 * 3) There are multiple errors and prior ID comparing fails;853 * We check AER status registers to find possible reporter.854 */855 if (atomic_read(&dev->enable_cnt) == 0)856 return false;857 858 /* Check if AER is enabled */859 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16);860 if (!(reg16 & PCI_EXP_AER_FLAGS))861 return false;862 863 if (!aer)864 return false;865 866 /* Check if error is recorded */867 if (e_info->severity == AER_CORRECTABLE) {868 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);869 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);870 } else {871 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);872 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);873 }874 if (status & ~mask)875 return true;876 877 return false;878}879 880static int find_device_iter(struct pci_dev *dev, void *data)881{882 struct aer_err_info *e_info = (struct aer_err_info *)data;883 884 if (is_error_source(dev, e_info)) {885 /* List this device */886 if (add_error_device(e_info, dev)) {887 /* We cannot handle more... Stop iteration */888 /* TODO: Should print error message here? */889 return 1;890 }891 892 /* If there is only a single error, stop iteration */893 if (!e_info->multi_error_valid)894 return 1;895 }896 return 0;897}898 899/**900 * find_source_device - search through device hierarchy for source device901 * @parent: pointer to Root Port pci_dev data structure902 * @e_info: including detailed error information such like id903 *904 * Return true if found.905 *906 * Invoked by DPC when error is detected at the Root Port.907 * Caller of this function must set id, severity, and multi_error_valid of908 * struct aer_err_info pointed by @e_info properly. This function must fill909 * e_info->error_dev_num and e_info->dev[], based on the given information.910 */911static bool find_source_device(struct pci_dev *parent,912 struct aer_err_info *e_info)913{914 struct pci_dev *dev = parent;915 int result;916 917 /* Must reset in this function */918 e_info->error_dev_num = 0;919 920 /* Is Root Port an agent that sends error message? */921 result = find_device_iter(dev, e_info);922 if (result)923 return true;924 925 if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC)926 pcie_walk_rcec(parent, find_device_iter, e_info);927 else928 pci_walk_bus(parent->subordinate, find_device_iter, e_info);929 930 if (!e_info->error_dev_num) {931 u8 bus = e_info->id >> 8;932 u8 devfn = e_info->id & 0xff;933 934 pci_info(parent, "found no error details for %04x:%02x:%02x.%d\n",935 pci_domain_nr(parent->bus), bus, PCI_SLOT(devfn),936 PCI_FUNC(devfn));937 return false;938 }939 return true;940}941 942#ifdef CONFIG_PCIEAER_CXL943 944/**945 * pci_aer_unmask_internal_errors - unmask internal errors946 * @dev: pointer to the pcie_dev data structure947 *948 * Unmasks internal errors in the Uncorrectable and Correctable Error949 * Mask registers.950 *951 * Note: AER must be enabled and supported by the device which must be952 * checked in advance, e.g. with pcie_aer_is_native().953 */954static void pci_aer_unmask_internal_errors(struct pci_dev *dev)955{956 int aer = dev->aer_cap;957 u32 mask;958 959 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);960 mask &= ~PCI_ERR_UNC_INTN;961 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask);962 963 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);964 mask &= ~PCI_ERR_COR_INTERNAL;965 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask);966}967 968static bool is_cxl_mem_dev(struct pci_dev *dev)969{970 /*971 * The capability, status, and control fields in Device 0,972 * Function 0 DVSEC control the CXL functionality of the973 * entire device (CXL 3.0, 8.1.3).974 */975 if (dev->devfn != PCI_DEVFN(0, 0))976 return false;977 978 /*979 * CXL Memory Devices must have the 502h class code set (CXL980 * 3.0, 8.1.12.1).981 */982 if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)983 return false;984 985 return true;986}987 988static bool cxl_error_is_native(struct pci_dev *dev)989{990 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);991 992 return (pcie_ports_native || host->native_aer);993}994 995static bool is_internal_error(struct aer_err_info *info)996{997 if (info->severity == AER_CORRECTABLE)998 return info->status & PCI_ERR_COR_INTERNAL;999 1000 return info->status & PCI_ERR_UNC_INTN;1001}1002 1003static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data)1004{1005 struct aer_err_info *info = (struct aer_err_info *)data;1006 const struct pci_error_handlers *err_handler;1007 1008 if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev))1009 return 0;1010 1011 /* protect dev->driver */1012 device_lock(&dev->dev);1013 1014 err_handler = dev->driver ? dev->driver->err_handler : NULL;1015 if (!err_handler)1016 goto out;1017 1018 if (info->severity == AER_CORRECTABLE) {1019 if (err_handler->cor_error_detected)1020 err_handler->cor_error_detected(dev);1021 } else if (err_handler->error_detected) {1022 if (info->severity == AER_NONFATAL)1023 err_handler->error_detected(dev, pci_channel_io_normal);1024 else if (info->severity == AER_FATAL)1025 err_handler->error_detected(dev, pci_channel_io_frozen);1026 }1027out:1028 device_unlock(&dev->dev);1029 return 0;1030}1031 1032static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info)1033{1034 /*1035 * Internal errors of an RCEC indicate an AER error in an1036 * RCH's downstream port. Check and handle them in the CXL.mem1037 * device driver.1038 */1039 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC &&1040 is_internal_error(info))1041 pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info);1042}1043 1044static int handles_cxl_error_iter(struct pci_dev *dev, void *data)1045{1046 bool *handles_cxl = data;1047 1048 if (!*handles_cxl)1049 *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev);1050 1051 /* Non-zero terminates iteration */1052 return *handles_cxl;1053}1054 1055static bool handles_cxl_errors(struct pci_dev *rcec)1056{1057 bool handles_cxl = false;1058 1059 if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC &&1060 pcie_aer_is_native(rcec))1061 pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl);1062 1063 return handles_cxl;1064}1065 1066static void cxl_rch_enable_rcec(struct pci_dev *rcec)1067{1068 if (!handles_cxl_errors(rcec))1069 return;1070 1071 pci_aer_unmask_internal_errors(rcec);1072 pci_info(rcec, "CXL: Internal errors unmasked");1073}1074 1075#else1076static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { }1077static inline void cxl_rch_handle_error(struct pci_dev *dev,1078 struct aer_err_info *info) { }1079#endif1080 1081/**1082 * pci_aer_handle_error - handle logging error into an event log1083 * @dev: pointer to pci_dev data structure of error source device1084 * @info: comprehensive error information1085 *1086 * Invoked when an error being detected by Root Port.1087 */1088static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info)1089{1090 int aer = dev->aer_cap;1091 1092 if (info->severity == AER_CORRECTABLE) {1093 /*1094 * Correctable error does not need software intervention.1095 * No need to go through error recovery process.1096 */1097 if (aer)1098 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS,1099 info->status);1100 if (pcie_aer_is_native(dev)) {1101 struct pci_driver *pdrv = dev->driver;1102 1103 if (pdrv && pdrv->err_handler &&1104 pdrv->err_handler->cor_error_detected)1105 pdrv->err_handler->cor_error_detected(dev);1106 pcie_clear_device_status(dev);1107 }1108 } else if (info->severity == AER_NONFATAL)1109 pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset);1110 else if (info->severity == AER_FATAL)1111 pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset);1112}1113 1114static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info)1115{1116 cxl_rch_handle_error(dev, info);1117 pci_aer_handle_error(dev, info);1118 pci_dev_put(dev);1119}1120 1121#ifdef CONFIG_ACPI_APEI_PCIEAER1122 1123#define AER_RECOVER_RING_SIZE 161124 1125struct aer_recover_entry {1126 u8 bus;1127 u8 devfn;1128 u16 domain;1129 int severity;1130 struct aer_capability_regs *regs;1131};1132 1133static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry,1134 AER_RECOVER_RING_SIZE);1135 1136static void aer_recover_work_func(struct work_struct *work)1137{1138 struct aer_recover_entry entry;1139 struct pci_dev *pdev;1140 1141 while (kfifo_get(&aer_recover_ring, &entry)) {1142 pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus,1143 entry.devfn);1144 if (!pdev) {1145 pr_err("no pci_dev for %04x:%02x:%02x.%x\n",1146 entry.domain, entry.bus,1147 PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn));1148 continue;1149 }1150 pci_print_aer(pdev, entry.severity, entry.regs);1151 /*1152 * Memory for aer_capability_regs(entry.regs) is being allocated from the1153 * ghes_estatus_pool to protect it from overwriting when multiple sections1154 * are present in the error status. Thus free the same after processing1155 * the data.1156 */1157 ghes_estatus_pool_region_free((unsigned long)entry.regs,1158 sizeof(struct aer_capability_regs));1159 1160 if (entry.severity == AER_NONFATAL)1161 pcie_do_recovery(pdev, pci_channel_io_normal,1162 aer_root_reset);1163 else if (entry.severity == AER_FATAL)1164 pcie_do_recovery(pdev, pci_channel_io_frozen,1165 aer_root_reset);1166 pci_dev_put(pdev);1167 }1168}1169 1170/*1171 * Mutual exclusion for writers of aer_recover_ring, reader side don't1172 * need lock, because there is only one reader and lock is not needed1173 * between reader and writer.1174 */1175static DEFINE_SPINLOCK(aer_recover_ring_lock);1176static DECLARE_WORK(aer_recover_work, aer_recover_work_func);1177 1178void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn,1179 int severity, struct aer_capability_regs *aer_regs)1180{1181 struct aer_recover_entry entry = {1182 .bus = bus,1183 .devfn = devfn,1184 .domain = domain,1185 .severity = severity,1186 .regs = aer_regs,1187 };1188 1189 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1,1190 &aer_recover_ring_lock))1191 schedule_work(&aer_recover_work);1192 else1193 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n",1194 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));1195}1196EXPORT_SYMBOL_GPL(aer_recover_queue);1197#endif1198 1199/**1200 * aer_get_device_error_info - read error status from dev and store it to info1201 * @dev: pointer to the device expected to have a error record1202 * @info: pointer to structure to store the error record1203 *1204 * Return 1 on success, 0 on error.1205 *1206 * Note that @info is reused among all error devices. Clear fields properly.1207 */1208int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)1209{1210 int type = pci_pcie_type(dev);1211 int aer = dev->aer_cap;1212 u32 aercc;1213 1214 /* Must reset in this function */1215 info->status = 0;1216 info->tlp_header_valid = 0;1217 1218 /* The device might not support AER */1219 if (!aer)1220 return 0;1221 1222 if (info->severity == AER_CORRECTABLE) {1223 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS,1224 &info->status);1225 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK,1226 &info->mask);1227 if (!(info->status & ~info->mask))1228 return 0;1229 } else if (type == PCI_EXP_TYPE_ROOT_PORT ||1230 type == PCI_EXP_TYPE_RC_EC ||1231 type == PCI_EXP_TYPE_DOWNSTREAM ||1232 info->severity == AER_NONFATAL) {1233 1234 /* Link is still healthy for IO reads */1235 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS,1236 &info->status);1237 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK,1238 &info->mask);1239 if (!(info->status & ~info->mask))1240 return 0;1241 1242 /* Get First Error Pointer */1243 pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc);1244 info->first_error = PCI_ERR_CAP_FEP(aercc);1245 1246 if (info->status & AER_LOG_TLP_MASKS) {1247 info->tlp_header_valid = 1;1248 pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, &info->tlp);1249 }1250 }1251 1252 return 1;1253}1254 1255static inline void aer_process_err_devices(struct aer_err_info *e_info)1256{1257 int i;1258 1259 /* Report all before handle them, not to lost records by reset etc. */1260 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {1261 if (aer_get_device_error_info(e_info->dev[i], e_info))1262 aer_print_error(e_info->dev[i], e_info);1263 }1264 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {1265 if (aer_get_device_error_info(e_info->dev[i], e_info))1266 handle_error_source(e_info->dev[i], e_info);1267 }1268}1269 1270/**1271 * aer_isr_one_error - consume an error detected by root port1272 * @rpc: pointer to the root port which holds an error1273 * @e_src: pointer to an error source1274 */1275static void aer_isr_one_error(struct aer_rpc *rpc,1276 struct aer_err_source *e_src)1277{1278 struct pci_dev *pdev = rpc->rpd;1279 struct aer_err_info e_info;1280 1281 pci_rootport_aer_stats_incr(pdev, e_src);1282 1283 /*1284 * There is a possibility that both correctable error and1285 * uncorrectable error being logged. Report correctable error first.1286 */1287 if (e_src->status & PCI_ERR_ROOT_COR_RCV) {1288 e_info.id = ERR_COR_ID(e_src->id);1289 e_info.severity = AER_CORRECTABLE;1290 1291 if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV)1292 e_info.multi_error_valid = 1;1293 else1294 e_info.multi_error_valid = 0;1295 aer_print_port_info(pdev, &e_info);1296 1297 if (find_source_device(pdev, &e_info))1298 aer_process_err_devices(&e_info);1299 }1300 1301 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {1302 e_info.id = ERR_UNCOR_ID(e_src->id);1303 1304 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)1305 e_info.severity = AER_FATAL;1306 else1307 e_info.severity = AER_NONFATAL;1308 1309 if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV)1310 e_info.multi_error_valid = 1;1311 else1312 e_info.multi_error_valid = 0;1313 1314 aer_print_port_info(pdev, &e_info);1315 1316 if (find_source_device(pdev, &e_info))1317 aer_process_err_devices(&e_info);1318 }1319}1320 1321/**1322 * aer_isr - consume errors detected by root port1323 * @irq: IRQ assigned to Root Port1324 * @context: pointer to Root Port data structure1325 *1326 * Invoked, as DPC, when root port records new detected error1327 */1328static irqreturn_t aer_isr(int irq, void *context)1329{1330 struct pcie_device *dev = (struct pcie_device *)context;1331 struct aer_rpc *rpc = get_service_data(dev);1332 struct aer_err_source e_src;1333 1334 if (kfifo_is_empty(&rpc->aer_fifo))1335 return IRQ_NONE;1336 1337 while (kfifo_get(&rpc->aer_fifo, &e_src))1338 aer_isr_one_error(rpc, &e_src);1339 return IRQ_HANDLED;1340}1341 1342/**1343 * aer_irq - Root Port's ISR1344 * @irq: IRQ assigned to Root Port1345 * @context: pointer to Root Port data structure1346 *1347 * Invoked when Root Port detects AER messages.1348 */1349static irqreturn_t aer_irq(int irq, void *context)1350{1351 struct pcie_device *pdev = (struct pcie_device *)context;1352 struct aer_rpc *rpc = get_service_data(pdev);1353 struct pci_dev *rp = rpc->rpd;1354 int aer = rp->aer_cap;1355 struct aer_err_source e_src = {};1356 1357 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status);1358 if (!(e_src.status & AER_ERR_STATUS_MASK))1359 return IRQ_NONE;1360 1361 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id);1362 pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status);1363 1364 if (!kfifo_put(&rpc->aer_fifo, e_src))1365 return IRQ_HANDLED;1366 1367 return IRQ_WAKE_THREAD;1368}1369 1370static void aer_enable_irq(struct pci_dev *pdev)1371{1372 int aer = pdev->aer_cap;1373 u32 reg32;1374 1375 /* Enable Root Port's interrupt in response to error messages */1376 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32);1377 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK;1378 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);1379}1380 1381static void aer_disable_irq(struct pci_dev *pdev)1382{1383 int aer = pdev->aer_cap;1384 u32 reg32;1385 1386 /* Disable Root's interrupt in response to error messages */1387 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32);1388 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;1389 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);1390}1391 1392/**1393 * aer_enable_rootport - enable Root Port's interrupts when receiving messages1394 * @rpc: pointer to a Root Port data structure1395 *1396 * Invoked when PCIe bus loads AER service driver.1397 */1398static void aer_enable_rootport(struct aer_rpc *rpc)1399{1400 struct pci_dev *pdev = rpc->rpd;1401 int aer = pdev->aer_cap;1402 u16 reg16;1403 u32 reg32;1404 1405 /* Clear PCIe Capability's Device Status */1406 pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16);1407 pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16);1408 1409 /* Disable system error generation in response to error messages */1410 pcie_capability_clear_word(pdev, PCI_EXP_RTCTL,1411 SYSTEM_ERROR_INTR_ON_MESG_MASK);1412 1413 /* Clear error status */1414 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32);1415 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);1416 pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32);1417 pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32);1418 pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32);1419 pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32);1420 1421 aer_enable_irq(pdev);1422}1423 1424/**1425 * aer_disable_rootport - disable Root Port's interrupts when receiving messages1426 * @rpc: pointer to a Root Port data structure1427 *1428 * Invoked when PCIe bus unloads AER service driver.1429 */1430static void aer_disable_rootport(struct aer_rpc *rpc)1431{1432 struct pci_dev *pdev = rpc->rpd;1433 int aer = pdev->aer_cap;1434 u32 reg32;1435 1436 aer_disable_irq(pdev);1437 1438 /* Clear Root's error status reg */1439 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32);1440 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);1441}1442 1443/**1444 * aer_remove - clean up resources1445 * @dev: pointer to the pcie_dev data structure1446 *1447 * Invoked when PCI Express bus unloads or AER probe fails.1448 */1449static void aer_remove(struct pcie_device *dev)1450{1451 struct aer_rpc *rpc = get_service_data(dev);1452 1453 aer_disable_rootport(rpc);1454}1455 1456/**1457 * aer_probe - initialize resources1458 * @dev: pointer to the pcie_dev data structure1459 *1460 * Invoked when PCI Express bus loads AER service driver.1461 */1462static int aer_probe(struct pcie_device *dev)1463{1464 int status;1465 struct aer_rpc *rpc;1466 struct device *device = &dev->device;1467 struct pci_dev *port = dev->port;1468 1469 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) <1470 AER_MAX_TYPEOF_COR_ERRS);1471 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) <1472 AER_MAX_TYPEOF_UNCOR_ERRS);1473 1474 /* Limit to Root Ports or Root Complex Event Collectors */1475 if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) &&1476 (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT))1477 return -ENODEV;1478 1479 rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL);1480 if (!rpc)1481 return -ENOMEM;1482 1483 rpc->rpd = port;1484 INIT_KFIFO(rpc->aer_fifo);1485 set_service_data(dev, rpc);1486 1487 status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr,1488 IRQF_SHARED, "aerdrv", dev);1489 if (status) {1490 pci_err(port, "request AER IRQ %d failed\n", dev->irq);1491 return status;1492 }1493 1494 cxl_rch_enable_rcec(port);1495 aer_enable_rootport(rpc);1496 pci_info(port, "enabled with IRQ %d\n", dev->irq);1497 return 0;1498}1499 1500static int aer_suspend(struct pcie_device *dev)1501{1502 struct aer_rpc *rpc = get_service_data(dev);1503 1504 aer_disable_rootport(rpc);1505 return 0;1506}1507 1508static int aer_resume(struct pcie_device *dev)1509{1510 struct aer_rpc *rpc = get_service_data(dev);1511 1512 aer_enable_rootport(rpc);1513 return 0;1514}1515 1516/**1517 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP1518 * @dev: pointer to Root Port, RCEC, or RCiEP1519 *1520 * Invoked by Port Bus driver when performing reset.1521 */1522static pci_ers_result_t aer_root_reset(struct pci_dev *dev)1523{1524 int type = pci_pcie_type(dev);1525 struct pci_dev *root;1526 int aer;1527 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);1528 u32 reg32;1529 int rc;1530 1531 /*1532 * Only Root Ports and RCECs have AER Root Command and Root Status1533 * registers. If "dev" is an RCiEP, the relevant registers are in1534 * the RCEC.1535 */1536 if (type == PCI_EXP_TYPE_RC_END)1537 root = dev->rcec;1538 else1539 root = pcie_find_root_port(dev);1540 1541 /*1542 * If the platform retained control of AER, an RCiEP may not have1543 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In1544 * that case, firmware is responsible for these registers.1545 */1546 aer = root ? root->aer_cap : 0;1547 1548 if ((host->native_aer || pcie_ports_native) && aer)1549 aer_disable_irq(root);1550 1551 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) {1552 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET);1553 if (!rc)1554 pci_info(dev, "has been reset\n");1555 else1556 pci_info(dev, "not reset (no FLR support: %d)\n", rc);1557 } else {1558 rc = pci_bus_error_reset(dev);1559 pci_info(dev, "%s Port link has been reset (%d)\n",1560 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc);1561 }1562 1563 if ((host->native_aer || pcie_ports_native) && aer) {1564 /* Clear Root Error Status */1565 pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, ®32);1566 pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32);1567 1568 aer_enable_irq(root);1569 }1570 1571 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;1572}1573 1574static struct pcie_port_service_driver aerdriver = {1575 .name = "aer",1576 .port_type = PCIE_ANY_PORT,1577 .service = PCIE_PORT_SERVICE_AER,1578 1579 .probe = aer_probe,1580 .suspend = aer_suspend,1581 .resume = aer_resume,1582 .remove = aer_remove,1583};1584 1585/**1586 * pcie_aer_init - register AER root service driver1587 *1588 * Invoked when AER root service driver is loaded.1589 */1590int __init pcie_aer_init(void)1591{1592 if (!pci_aer_available())1593 return -ENXIO;1594 return pcie_port_service_register(&aerdriver);1595}1596