From 2e560133d42deb7fd712c84c11f1ed9aa38002e1 Mon Sep 17 00:00:00 2001 From: Ruidong Tian Date: Thu, 27 Mar 2025 17:27:38 +0800 Subject: [PATCH 1/2] aer: print pci device name and vendor/device id New aer log like follow: <...>-2682840 [125] .... 0.017661 aer_event 2025-03-27 17:34:44 +0800 0000:99:00.0 (Intel Corporation Device 0b60 - vendor_id: 0x8086 device_id: 0xb60) Data Link Protocol Uncorrected (Non-Fatal) Signed-off-by: Ruidong Tian --- Makefile.am | 4 ++-- configure.ac | 8 ++++++++ misc/rasdaemon.spec.in | 2 ++ ras-aer-handler.c | 46 +++++++++++++++++++++++++++++++++++++++++- ras-record.h | 2 +- 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index 01132fec..264cef59 100644 --- a/Makefile.am +++ b/Makefile.am @@ -91,8 +91,8 @@ if WITH_JAGUAR_NS_DECODE rasdaemon_SOURCES += non-standard-jaguarmicro.c endif -rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS) -rasdaemon_CFLAGS = $(SQLITE3_CFLAGS) $(LIBTRACEEVENT_CFLAGS) +rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS) $(LIBPCI_LIBS) +rasdaemon_CFLAGS = $(SQLITE3_CFLAGS) $(LIBTRACEEVENT_CFLAGS) $(LIBPCI_CFLAGS) include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \ ras-aer-handler.h ras-mce-handler.h ras-record.h bitfield.h ras-report.h \ diff --git a/configure.ac b/configure.ac index 1cb00b6d..77118631 100644 --- a/configure.ac +++ b/configure.ac @@ -54,6 +54,14 @@ AC_ARG_ENABLE([aer], AS_IF([test "x$enable_aer" = "xyes" || test "x$enable_all" = "xyes"], [ AC_DEFINE(HAVE_AER,1,"have PCIe AER events collect") AC_SUBST([WITH_AER]) + + has_libpci_ver=0 + dnl check for pciutils library + PKG_CHECK_MODULES([LIBPCI], [libpci], [has_libpci_ver=1]) + + AS_IF([test "$has_libpci_ver" -eq 0], [ + AC_MSG_ERROR([libpci is required but were not found]) +]) ]) AM_CONDITIONAL([WITH_AER], [test x$enable_aer = xyes || test x$enable_all = xyes]) AM_COND_IF([WITH_AER], [USE_AER="yes"], [USE_AER="no"]) diff --git a/misc/rasdaemon.spec.in b/misc/rasdaemon.spec.in index 32c69b74..4b3e129a 100644 --- a/misc/rasdaemon.spec.in +++ b/misc/rasdaemon.spec.in @@ -17,10 +17,12 @@ BuildRequires: perl-generators BuildRequires: sqlite-devel BuildRequires: systemd BuildRequires: libtraceevent-devel +BuildRequires: pciutils-devel Provides: bundled(kernel-event-lib) Requires: hwdata Requires: perl-DBD-SQLite Requires: libtraceevent +Requires: pciutils-devel %ifarch %{ix86} x86_64 Requires: dmidecode %endif diff --git a/ras-aer-handler.c b/ras-aer-handler.c index 5d069f3f..53acbc83 100644 --- a/ras-aer-handler.c +++ b/ras-aer-handler.c @@ -4,6 +4,7 @@ * Copyright (C) 2013 Mauro Carvalho Chehab */ +#include #include #include #include @@ -63,6 +64,45 @@ void ras_aer_handler_init(int enable_ipmitool) #define BUF_LEN 1024 +static void get_pci_dev_name(char *bdf, char *pci_name, ssize_t len, u16 *vendor_id, u16 *device_id) +{ + struct pci_access *pacc; + struct pci_dev *dev; + struct pci_filter filter = {0}; + char *err; + + if (!pci_name) + return; + + pacc = pci_alloc(); + if (!pacc) + return; + + pci_init(pacc); + pci_scan_bus(pacc); + pci_filter_init(pacc, &filter); + err = pci_filter_parse_slot(&filter, bdf); + if (err) { + log(TERM, LOG_ERR, "Invalid PCI device name %s\n", bdf); + goto free; + } + + for (dev = pacc->devices; dev; dev = dev->next) { + if (pci_filter_match(&filter, dev)) { + pci_fill_info(dev, PCI_FILL_IDENT); + *vendor_id = dev->vendor_id; + *device_id = dev->device_id; + pci_lookup_name(pacc, pci_name, len, + PCI_LOOKUP_VENDOR | PCI_LOOKUP_DEVICE, + dev->vendor_id, dev->device_id); + break; + } + } + +free: + pci_cleanup(pacc); +} + int ras_aer_event_handler(struct trace_seq *s, struct tep_record *record, struct tep_event *event, void *context) @@ -75,7 +115,8 @@ int ras_aer_event_handler(struct trace_seq *s, time_t now; struct tm *tm; struct ras_aer_event ev; - char buf[BUF_LEN]; + char buf[BUF_LEN] = { 0 }; + uint16_t vendor_id = 0, device_id = 0; #ifdef HAVE_AMP_NS_DECODE char ipmi_add_sel[105]; uint8_t sel_data[5]; @@ -108,6 +149,9 @@ int ras_aer_event_handler(struct trace_seq *s, return -1; trace_seq_printf(s, "%s ", ev.dev_name); + get_pci_dev_name(ev.dev_name, buf, sizeof(buf), &vendor_id, &device_id); + trace_seq_printf(s, "(%s - vendor_id: %#x device_id: %#x) ", buf, vendor_id, device_id); + if (tep_get_field_val(s, event, "status", record, &status_val, 1) < 0) return -1; diff --git a/ras-record.h b/ras-record.h index eec07027..ed8a71ee 100644 --- a/ras-record.h +++ b/ras-record.h @@ -41,7 +41,7 @@ struct ras_mc_offline_event { struct ras_aer_event { char timestamp[64]; const char *error_type; - const char *dev_name; + char *dev_name; uint8_t tlp_header_valid; uint32_t *tlp_header; const char *msg; From b8cefed2262ccccedef880d558b6d694680c768f Mon Sep 17 00:00:00 2001 From: Ruidong Tian Date: Thu, 27 Mar 2025 17:45:16 +0800 Subject: [PATCH 2/2] rasdaemon: introduce EDPC config in rasdaemon System with EDPC enabled device can recovery from fatal aer error. Rasdaemon now helps users correctly configure EDPC functionality. Rasdaemon will enable EDPC for fatal error if PCIE_EDPC_ENABLE set to 1. All device with EDPC capability will be enabled by default if EDPC_DEVICE is specified, only the specified device will be enabled. For example: PCIE_EDPC_ENABLE=1 EDPC_DEVICE=0000:01:00.0 only enable device 0000:01:00.0. Signed-off-by: Ruidong Tian --- Makefile.am | 4 +- misc/rasdaemon.env | 11 +++ ras-pcie-edpc.c | 217 +++++++++++++++++++++++++++++++++++++++++++++ ras-pcie-edpc.h | 9 ++ rasdaemon.c | 7 ++ 5 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 ras-pcie-edpc.c create mode 100644 ras-pcie-edpc.h diff --git a/Makefile.am b/Makefile.am index 264cef59..ab296dd3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -34,7 +34,7 @@ if WITH_SQLITE3 rasdaemon_SOURCES += ras-record.c endif if WITH_AER - rasdaemon_SOURCES += ras-aer-handler.c + rasdaemon_SOURCES += ras-aer-handler.c ras-pcie-edpc.c endif if WITH_NON_STANDARD rasdaemon_SOURCES += ras-non-standard-handler.c @@ -100,7 +100,7 @@ include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \ ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h \ non-standard-hisilicon.h non-standard-ampere.h ras-memory-failure-handler.h \ ras-cxl-handler.h ras-cpu-isolation.h queue.h non-standard-yitian.h \ - non-standard-jaguarmicro.h trigger.h unified-sel.h + non-standard-jaguarmicro.h trigger.h unified-sel.h ras-pcie-edpc.h # This rule can't be called with more than one Makefile job (like make -j8) # I can't figure out a way to fix that diff --git a/misc/rasdaemon.env b/misc/rasdaemon.env index 963aaa05..10aa63d6 100644 --- a/misc/rasdaemon.env +++ b/misc/rasdaemon.env @@ -88,3 +88,14 @@ TRIGGER_DIR= # MC_UE_TRIGGER=mc_event_trigger MC_CE_TRIGGER= MC_UE_TRIGGER= + +# EDPC config +# +# rasdaemon will enable EDPC for fatal error if PCIE_EDPC_ENABLE set to 1 +# All device with EDPC capability will be enabled by default, +# if EDPC_DEVICE is specified, only the specified device will be enabled +# For example: +# PCIE_EDPC_ENABLE=1 +# EDPC_DEVICE=0000:01:00.0 // only enable device 0000:01:00.0 +PCIE_EDPC_ENABLE=0 +EDPC_DEVICE= diff --git a/ras-pcie-edpc.c b/ras-pcie-edpc.c new file mode 100644 index 00000000..4731b05a --- /dev/null +++ b/ras-pcie-edpc.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* + * Copyright (C) 2025 Alibaba Inc + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ras-pcie-edpc.h" +#include "ras-logger.h" +#include "types.h" + +#define EDPC_DEVICE "EDPC_DEVICE" + +#define PCI_EXP_DPC_CTL_EN_MASK 0x3 + +static char *edpc_str[] = { + [PCI_EXP_DPC_CTL_EN_FATAL] = "Fatal Error", + [PCI_EXP_DPC_CTL_EN_NONFATAL] = "Non-Fatal Error", +}; + +static bool is_cxl_mem_or_cache(struct pci_dev *dev) +{ + struct pci_cap *cap; + u32 hdr; + u16 vendor, cxl_cap, id; + + cap = pci_find_cap(dev, PCI_EXT_CAP_ID_DVSEC, PCI_CAP_EXTENDED); + if (!cap) + return false; + + hdr = pci_read_long(dev, cap->addr + PCI_DVSEC_HEADER1); + vendor = hdr & GENMASK(15, 0); + id = pci_read_word(dev, cap->addr + PCI_DVSEC_HEADER2); + if (vendor != PCI_DVSEC_VENDOR_ID_CXL || id != PCI_DVSEC_ID_CXL) + return false; + + cxl_cap = pci_read_word(dev, cap->addr + PCI_CXL_CAP); + if (cxl_cap & (PCI_CXL_CAP_CACHE | PCI_CXL_CAP_MEM)) + return true; + + return false; +} + +/** + * CXL 2.0 RAS spec: 4.2: + * Enabling eDPC is not recommended in most CXL 2.0 systems because eDPC + * containment flow brings the link down, disrupting CXL.cache and + * CXL.mem traffic which can lead to host timeouts. + */ +static void cxl_check_rp(struct pci_dev *dev, struct pci_dev *dpc) +{ + struct pci_dev *dev_p, *dpc_p; + for (dev_p = dev->parent; dev_p; dev_p = dev_p->parent) { + for (dpc_p = dpc->next; dpc_p; dpc_p = dpc_p->next) { + if (dev_p->domain == dpc_p->domain && + dev_p->bus == dpc_p->bus && + dev_p->dev == dpc_p->dev && + dev_p->func == dpc_p->func) { + dpc_p->aux = (void *)true; + log(TERM, LOG_INFO, "Device %x:%x:%x.%x is CXL RP, ignore EDPC config\n", + dpc_p->domain, dpc_p->bus, dpc_p->dev, dpc_p->func); + } + } + } +} + +static bool has_edpc(struct pci_dev *dev) +{ + struct pci_cap *cap; + + pci_fill_info(dev, PCI_FILL_EXT_CAPS); + cap = pci_find_cap(dev, PCI_EXT_CAP_ID_DPC, PCI_CAP_EXTENDED); + if (!cap) + return false; + return true; +} + +static void set_edpc(struct pci_dev *dev) +{ + struct pci_cap *cap; + u16 control; + int need_config = 0; + + cap = pci_find_cap(dev, PCI_EXT_CAP_ID_DPC, PCI_CAP_EXTENDED); + if (!cap) + return; + + control = pci_read_word(dev, cap->addr + PCI_EXP_DPC_CTL); + need_config = PCI_DPC_CTL_TRIGGER(control) == PCI_EXP_DPC_CTL_EN_FATAL ? 0 : 1; + log(TERM, LOG_INFO, "Device %x:%x:%x.%x origin EDPC %s and triggered for %s, %s need config\n", + dev->domain, dev->bus, dev->dev, dev->func, + (control & PCI_EXP_DPC_CTL_INT_EN) ? "enabled" : "disabled", + edpc_str[control & PCI_EXP_DPC_CTL_EN_MASK], + need_config ? "" : "not"); + + if (need_config) { + control &= PCI_EXP_DPC_CTL_EN_MASK; + control |= PCI_EXP_DPC_CTL_EN_FATAL; + pci_write_word(dev, cap->addr + PCI_EXP_DPC_CTL, control); + log(TERM, LOG_INFO, "Device %x:%x:%x.%x EDPC %s and triggered for %s\n", + dev->domain, dev->bus, dev->dev, dev->func, + (control & PCI_EXP_DPC_CTL_INT_EN) ? "enabled" : "disabled", + edpc_str[control & PCI_EXP_DPC_CTL_EN_MASK]); + } +} + +static struct pci_filter *config_pcie_edpc_device(struct pci_access *pacc, char *names, int *len) +{ + int i; + struct pci_filter *filter = NULL; + char *token, *err, pci_names[MAX_PATH + 1]; + + strscpy(pci_names, names, sizeof(pci_names)); + for (i = 0; pci_names[i] != '\0'; i++) + if (pci_names[i] == ',') + (*len)++; + + filter = calloc(*len, sizeof(struct pci_filter)); + if (!filter) + return NULL; + + i = 0; + token = strtok(pci_names, ","); + while (token) { + pci_filter_init(pacc, &filter[i]); + err = pci_filter_parse_slot(&filter[i++], token); + if (err) { + free(filter); + log(TERM, LOG_ERR, "Invalid PCI device name %s\n", err); + return NULL; + } + token = strtok(NULL, ","); + } + + log(TERM, LOG_ERR, "Config PCIE EDPC for: %s\n", names); + + return filter; +} + +int config_pcie_edpc(void) +{ + struct pci_access *pacc; + struct pci_dev *dev, *dev_head, *tmp; + int ret = 0, len = 1, i; + char *pci_names; + struct pci_filter *filter = NULL; + struct pci_dev dev_dpc_head = { 0 }; + + pacc = pci_alloc(); + if (!pacc) + return -1; + + pci_init(pacc); + pci_scan_bus(pacc); + + pci_names = getenv(EDPC_DEVICE); + if (pci_names && strlen(pci_names) != 0) { + filter = config_pcie_edpc_device(pacc, pci_names, &len); + if (!filter) + goto free; + } else { + len = 0; + } + + dev_head = pacc->devices; + for (dev = dev_head; dev; dev = dev->next) { + pci_fill_info(dev, PCI_FILL_PARENT); + if (has_edpc(dev)) { + tmp = malloc(sizeof(struct pci_dev)); + if (!tmp) { + ret = -1; + goto free; + } + + memcpy(tmp, dev, sizeof(struct pci_dev)); + tmp->next = dev_dpc_head.next; + dev_dpc_head.next = tmp; + } + } + + for (dev = dev_head; dev; dev = dev->next) + if (is_cxl_mem_or_cache(dev)) + cxl_check_rp(dev, &dev_dpc_head); + + for (dev = dev_dpc_head.next; dev; dev = dev->next) { + if (!dev->aux) { + if (len) { + for (i = 0; i < len; i++) { + if (pci_filter_match(&filter[i], dev)) { + set_edpc(dev); + break; + } + } + } else { + set_edpc(dev); + } + } + } + +free: + while (dev_dpc_head.next) { + tmp = dev_dpc_head.next; + dev_dpc_head.next = tmp->next; + free(tmp); + } + + pci_cleanup(pacc); + free(filter); + return ret; +} diff --git a/ras-pcie-edpc.h b/ras-pcie-edpc.h new file mode 100644 index 00000000..a7b96a4f --- /dev/null +++ b/ras-pcie-edpc.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* + * Copyright (C) 2025 Alibaba Inc + */ + + #define PCIE_EDPC_ENABLE "PCIE_EDPC_ENABLE" + +int config_pcie_edpc(void); diff --git a/rasdaemon.c b/rasdaemon.c index 840be61b..19c9d821 100644 --- a/rasdaemon.c +++ b/rasdaemon.c @@ -8,11 +8,13 @@ #include #include #include +#include #include #include "ras-events.h" #include "ras-logger.h" #include "ras-record.h" +#include "ras-pcie-edpc.h" #include "types.h" /* @@ -209,6 +211,11 @@ int main(int argc, char *argv[]) if (daemon(0, 0)) exit(EXIT_FAILURE); + if (getenv(PCIE_EDPC_ENABLE) && atoi(getenv(PCIE_EDPC_ENABLE))) + config_pcie_edpc(); + else + log(TERM, LOG_INFO, "PCIE EDPC config is not enabled\n"); + handle_ras_events(args.record_events, args.enable_ipmitool); return 0;