diff --git a/Makefile.am b/Makefile.am index 01132fec..ab296dd3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -34,7 +34,7 @@ if WITH_SQLITE3 rasdaemon_SOURCES += ras-record.c endif if WITH_AER - rasdaemon_SOURCES += ras-aer-handler.c + rasdaemon_SOURCES += ras-aer-handler.c ras-pcie-edpc.c endif if WITH_NON_STANDARD rasdaemon_SOURCES += ras-non-standard-handler.c @@ -91,8 +91,8 @@ if WITH_JAGUAR_NS_DECODE rasdaemon_SOURCES += non-standard-jaguarmicro.c endif -rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS) -rasdaemon_CFLAGS = $(SQLITE3_CFLAGS) $(LIBTRACEEVENT_CFLAGS) +rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS) $(LIBPCI_LIBS) +rasdaemon_CFLAGS = $(SQLITE3_CFLAGS) $(LIBTRACEEVENT_CFLAGS) $(LIBPCI_CFLAGS) include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \ ras-aer-handler.h ras-mce-handler.h ras-record.h bitfield.h ras-report.h \ @@ -100,7 +100,7 @@ include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \ ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h \ non-standard-hisilicon.h non-standard-ampere.h ras-memory-failure-handler.h \ ras-cxl-handler.h ras-cpu-isolation.h queue.h non-standard-yitian.h \ - non-standard-jaguarmicro.h trigger.h unified-sel.h + non-standard-jaguarmicro.h trigger.h unified-sel.h ras-pcie-edpc.h # This rule can't be called with more than one Makefile job (like make -j8) # I can't figure out a way to fix that diff --git a/configure.ac b/configure.ac index 1cb00b6d..77118631 100644 --- a/configure.ac +++ b/configure.ac @@ -54,6 +54,14 @@ AC_ARG_ENABLE([aer], AS_IF([test "x$enable_aer" = "xyes" || test "x$enable_all" = "xyes"], [ AC_DEFINE(HAVE_AER,1,"have PCIe AER events collect") AC_SUBST([WITH_AER]) + + has_libpci_ver=0 + dnl check for pciutils library + PKG_CHECK_MODULES([LIBPCI], [libpci], [has_libpci_ver=1]) + + AS_IF([test "$has_libpci_ver" -eq 0], [ + AC_MSG_ERROR([libpci is required but were not found]) +]) ]) AM_CONDITIONAL([WITH_AER], [test x$enable_aer = xyes || test x$enable_all = xyes]) AM_COND_IF([WITH_AER], [USE_AER="yes"], [USE_AER="no"]) diff --git a/misc/rasdaemon.env b/misc/rasdaemon.env index 963aaa05..10aa63d6 100644 --- a/misc/rasdaemon.env +++ b/misc/rasdaemon.env @@ -88,3 +88,14 @@ TRIGGER_DIR= # MC_UE_TRIGGER=mc_event_trigger MC_CE_TRIGGER= MC_UE_TRIGGER= + +# EDPC config +# +# rasdaemon will enable EDPC for fatal error if PCIE_EDPC_ENABLE set to 1 +# All device with EDPC capability will be enabled by default, +# if EDPC_DEVICE is specified, only the specified device will be enabled +# For example: +# PCIE_EDPC_ENABLE=1 +# EDPC_DEVICE=0000:01:00.0 // only enable device 0000:01:00.0 +PCIE_EDPC_ENABLE=0 +EDPC_DEVICE= diff --git a/misc/rasdaemon.spec.in b/misc/rasdaemon.spec.in index 32c69b74..4b3e129a 100644 --- a/misc/rasdaemon.spec.in +++ b/misc/rasdaemon.spec.in @@ -17,10 +17,12 @@ BuildRequires: perl-generators BuildRequires: sqlite-devel BuildRequires: systemd BuildRequires: libtraceevent-devel +BuildRequires: pciutils-devel Provides: bundled(kernel-event-lib) Requires: hwdata Requires: perl-DBD-SQLite Requires: libtraceevent +Requires: pciutils-devel %ifarch %{ix86} x86_64 Requires: dmidecode %endif diff --git a/ras-aer-handler.c b/ras-aer-handler.c index 5d069f3f..53acbc83 100644 --- a/ras-aer-handler.c +++ b/ras-aer-handler.c @@ -4,6 +4,7 @@ * Copyright (C) 2013 Mauro Carvalho Chehab */ +#include #include #include #include @@ -63,6 +64,45 @@ void ras_aer_handler_init(int enable_ipmitool) #define BUF_LEN 1024 +static void get_pci_dev_name(char *bdf, char *pci_name, ssize_t len, u16 *vendor_id, u16 *device_id) +{ + struct pci_access *pacc; + struct pci_dev *dev; + struct pci_filter filter = {0}; + char *err; + + if (!pci_name) + return; + + pacc = pci_alloc(); + if (!pacc) + return; + + pci_init(pacc); + pci_scan_bus(pacc); + pci_filter_init(pacc, &filter); + err = pci_filter_parse_slot(&filter, bdf); + if (err) { + log(TERM, LOG_ERR, "Invalid PCI device name %s\n", bdf); + goto free; + } + + for (dev = pacc->devices; dev; dev = dev->next) { + if (pci_filter_match(&filter, dev)) { + pci_fill_info(dev, PCI_FILL_IDENT); + *vendor_id = dev->vendor_id; + *device_id = dev->device_id; + pci_lookup_name(pacc, pci_name, len, + PCI_LOOKUP_VENDOR | PCI_LOOKUP_DEVICE, + dev->vendor_id, dev->device_id); + break; + } + } + +free: + pci_cleanup(pacc); +} + int ras_aer_event_handler(struct trace_seq *s, struct tep_record *record, struct tep_event *event, void *context) @@ -75,7 +115,8 @@ int ras_aer_event_handler(struct trace_seq *s, time_t now; struct tm *tm; struct ras_aer_event ev; - char buf[BUF_LEN]; + char buf[BUF_LEN] = { 0 }; + uint16_t vendor_id = 0, device_id = 0; #ifdef HAVE_AMP_NS_DECODE char ipmi_add_sel[105]; uint8_t sel_data[5]; @@ -108,6 +149,9 @@ int ras_aer_event_handler(struct trace_seq *s, return -1; trace_seq_printf(s, "%s ", ev.dev_name); + get_pci_dev_name(ev.dev_name, buf, sizeof(buf), &vendor_id, &device_id); + trace_seq_printf(s, "(%s - vendor_id: %#x device_id: %#x) ", buf, vendor_id, device_id); + if (tep_get_field_val(s, event, "status", record, &status_val, 1) < 0) return -1; diff --git a/ras-pcie-edpc.c b/ras-pcie-edpc.c new file mode 100644 index 00000000..4731b05a --- /dev/null +++ b/ras-pcie-edpc.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* + * Copyright (C) 2025 Alibaba Inc + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ras-pcie-edpc.h" +#include "ras-logger.h" +#include "types.h" + +#define EDPC_DEVICE "EDPC_DEVICE" + +#define PCI_EXP_DPC_CTL_EN_MASK 0x3 + +static char *edpc_str[] = { + [PCI_EXP_DPC_CTL_EN_FATAL] = "Fatal Error", + [PCI_EXP_DPC_CTL_EN_NONFATAL] = "Non-Fatal Error", +}; + +static bool is_cxl_mem_or_cache(struct pci_dev *dev) +{ + struct pci_cap *cap; + u32 hdr; + u16 vendor, cxl_cap, id; + + cap = pci_find_cap(dev, PCI_EXT_CAP_ID_DVSEC, PCI_CAP_EXTENDED); + if (!cap) + return false; + + hdr = pci_read_long(dev, cap->addr + PCI_DVSEC_HEADER1); + vendor = hdr & GENMASK(15, 0); + id = pci_read_word(dev, cap->addr + PCI_DVSEC_HEADER2); + if (vendor != PCI_DVSEC_VENDOR_ID_CXL || id != PCI_DVSEC_ID_CXL) + return false; + + cxl_cap = pci_read_word(dev, cap->addr + PCI_CXL_CAP); + if (cxl_cap & (PCI_CXL_CAP_CACHE | PCI_CXL_CAP_MEM)) + return true; + + return false; +} + +/** + * CXL 2.0 RAS spec: 4.2: + * Enabling eDPC is not recommended in most CXL 2.0 systems because eDPC + * containment flow brings the link down, disrupting CXL.cache and + * CXL.mem traffic which can lead to host timeouts. + */ +static void cxl_check_rp(struct pci_dev *dev, struct pci_dev *dpc) +{ + struct pci_dev *dev_p, *dpc_p; + for (dev_p = dev->parent; dev_p; dev_p = dev_p->parent) { + for (dpc_p = dpc->next; dpc_p; dpc_p = dpc_p->next) { + if (dev_p->domain == dpc_p->domain && + dev_p->bus == dpc_p->bus && + dev_p->dev == dpc_p->dev && + dev_p->func == dpc_p->func) { + dpc_p->aux = (void *)true; + log(TERM, LOG_INFO, "Device %x:%x:%x.%x is CXL RP, ignore EDPC config\n", + dpc_p->domain, dpc_p->bus, dpc_p->dev, dpc_p->func); + } + } + } +} + +static bool has_edpc(struct pci_dev *dev) +{ + struct pci_cap *cap; + + pci_fill_info(dev, PCI_FILL_EXT_CAPS); + cap = pci_find_cap(dev, PCI_EXT_CAP_ID_DPC, PCI_CAP_EXTENDED); + if (!cap) + return false; + return true; +} + +static void set_edpc(struct pci_dev *dev) +{ + struct pci_cap *cap; + u16 control; + int need_config = 0; + + cap = pci_find_cap(dev, PCI_EXT_CAP_ID_DPC, PCI_CAP_EXTENDED); + if (!cap) + return; + + control = pci_read_word(dev, cap->addr + PCI_EXP_DPC_CTL); + need_config = PCI_DPC_CTL_TRIGGER(control) == PCI_EXP_DPC_CTL_EN_FATAL ? 0 : 1; + log(TERM, LOG_INFO, "Device %x:%x:%x.%x origin EDPC %s and triggered for %s, %s need config\n", + dev->domain, dev->bus, dev->dev, dev->func, + (control & PCI_EXP_DPC_CTL_INT_EN) ? "enabled" : "disabled", + edpc_str[control & PCI_EXP_DPC_CTL_EN_MASK], + need_config ? "" : "not"); + + if (need_config) { + control &= PCI_EXP_DPC_CTL_EN_MASK; + control |= PCI_EXP_DPC_CTL_EN_FATAL; + pci_write_word(dev, cap->addr + PCI_EXP_DPC_CTL, control); + log(TERM, LOG_INFO, "Device %x:%x:%x.%x EDPC %s and triggered for %s\n", + dev->domain, dev->bus, dev->dev, dev->func, + (control & PCI_EXP_DPC_CTL_INT_EN) ? "enabled" : "disabled", + edpc_str[control & PCI_EXP_DPC_CTL_EN_MASK]); + } +} + +static struct pci_filter *config_pcie_edpc_device(struct pci_access *pacc, char *names, int *len) +{ + int i; + struct pci_filter *filter = NULL; + char *token, *err, pci_names[MAX_PATH + 1]; + + strscpy(pci_names, names, sizeof(pci_names)); + for (i = 0; pci_names[i] != '\0'; i++) + if (pci_names[i] == ',') + (*len)++; + + filter = calloc(*len, sizeof(struct pci_filter)); + if (!filter) + return NULL; + + i = 0; + token = strtok(pci_names, ","); + while (token) { + pci_filter_init(pacc, &filter[i]); + err = pci_filter_parse_slot(&filter[i++], token); + if (err) { + free(filter); + log(TERM, LOG_ERR, "Invalid PCI device name %s\n", err); + return NULL; + } + token = strtok(NULL, ","); + } + + log(TERM, LOG_ERR, "Config PCIE EDPC for: %s\n", names); + + return filter; +} + +int config_pcie_edpc(void) +{ + struct pci_access *pacc; + struct pci_dev *dev, *dev_head, *tmp; + int ret = 0, len = 1, i; + char *pci_names; + struct pci_filter *filter = NULL; + struct pci_dev dev_dpc_head = { 0 }; + + pacc = pci_alloc(); + if (!pacc) + return -1; + + pci_init(pacc); + pci_scan_bus(pacc); + + pci_names = getenv(EDPC_DEVICE); + if (pci_names && strlen(pci_names) != 0) { + filter = config_pcie_edpc_device(pacc, pci_names, &len); + if (!filter) + goto free; + } else { + len = 0; + } + + dev_head = pacc->devices; + for (dev = dev_head; dev; dev = dev->next) { + pci_fill_info(dev, PCI_FILL_PARENT); + if (has_edpc(dev)) { + tmp = malloc(sizeof(struct pci_dev)); + if (!tmp) { + ret = -1; + goto free; + } + + memcpy(tmp, dev, sizeof(struct pci_dev)); + tmp->next = dev_dpc_head.next; + dev_dpc_head.next = tmp; + } + } + + for (dev = dev_head; dev; dev = dev->next) + if (is_cxl_mem_or_cache(dev)) + cxl_check_rp(dev, &dev_dpc_head); + + for (dev = dev_dpc_head.next; dev; dev = dev->next) { + if (!dev->aux) { + if (len) { + for (i = 0; i < len; i++) { + if (pci_filter_match(&filter[i], dev)) { + set_edpc(dev); + break; + } + } + } else { + set_edpc(dev); + } + } + } + +free: + while (dev_dpc_head.next) { + tmp = dev_dpc_head.next; + dev_dpc_head.next = tmp->next; + free(tmp); + } + + pci_cleanup(pacc); + free(filter); + return ret; +} diff --git a/ras-pcie-edpc.h b/ras-pcie-edpc.h new file mode 100644 index 00000000..a7b96a4f --- /dev/null +++ b/ras-pcie-edpc.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* + * Copyright (C) 2025 Alibaba Inc + */ + + #define PCIE_EDPC_ENABLE "PCIE_EDPC_ENABLE" + +int config_pcie_edpc(void); diff --git a/ras-record.h b/ras-record.h index eec07027..ed8a71ee 100644 --- a/ras-record.h +++ b/ras-record.h @@ -41,7 +41,7 @@ struct ras_mc_offline_event { struct ras_aer_event { char timestamp[64]; const char *error_type; - const char *dev_name; + char *dev_name; uint8_t tlp_header_valid; uint32_t *tlp_header; const char *msg; diff --git a/rasdaemon.c b/rasdaemon.c index 840be61b..19c9d821 100644 --- a/rasdaemon.c +++ b/rasdaemon.c @@ -8,11 +8,13 @@ #include #include #include +#include #include #include "ras-events.h" #include "ras-logger.h" #include "ras-record.h" +#include "ras-pcie-edpc.h" #include "types.h" /* @@ -209,6 +211,11 @@ int main(int argc, char *argv[]) if (daemon(0, 0)) exit(EXIT_FAILURE); + if (getenv(PCIE_EDPC_ENABLE) && atoi(getenv(PCIE_EDPC_ENABLE))) + config_pcie_edpc(); + else + log(TERM, LOG_INFO, "PCIE EDPC config is not enabled\n"); + handle_ras_events(args.record_events, args.enable_ipmitool); return 0;