From 43a78ea9f65841234d6b5bbde513a304a74e2902 Mon Sep 17 00:00:00 2001 From: William Roche Date: Tue, 9 Nov 2021 10:40:42 +0200 Subject: [PATCH 01/61] ipmitool to be used with AER only on Ampere Altra platforms When monitoring AER errors, verify if we are on an Altra platform to use ipmitool when dealing with this type of errors. Needs both --enable-aer and --enable-amp-ns-decode Fixes: 738bafafdcb2 ("Add error handling for Ampere-specific errors.") Signed-off-by: William Roche --- ras-aer-handler.c | 124 +++++++++++++++++++++++++++++++++++----------- ras-aer-handler.h | 1 + ras-events.c | 1 + 3 files changed, 97 insertions(+), 29 deletions(-) diff --git a/ras-aer-handler.c b/ras-aer-handler.c index 6f4cb2bb..24872f17 100644 --- a/ras-aer-handler.c +++ b/ras-aer-handler.c @@ -25,6 +25,11 @@ #include "ras-logger.h" #include "bitfield.h" #include "ras-report.h" +#ifdef HAVE_AMP_NS_DECODE +#include +#include +#include +#endif /* bit field meaning for correctable error */ static const char *aer_cor_errors[32] = { @@ -52,6 +57,86 @@ static const char *aer_uncor_errors[32] = { [20] = "Unsupported Request", }; +#ifdef HAVE_AMP_NS_DECODE +#define IPMITOOL_CMD "/usr/bin/ipmitool" +#define DMIDECODE_CMD "/usr/sbin/dmidecode" +static bool ampere_ipmitool = false; + +static void ras_report_aer_ipmi_init(void) +{ + struct utsname unm; + struct stat st; + int rc; + + /* + * Verify on startup if we are on an Ampere Altra or Altra Max + * platform, to set the use of ipmitool (if installed). + */ + if (stat(IPMITOOL_CMD, &st) != 0) + return; + + if ((uname(&unm) != 0) || (strncmp(unm.machine, "aarch64", 8) != 0)) + return; + + /* prefer dmidecode as lscpu may not have the necessary dmi info */ + if (stat(DMIDECODE_CMD, &st) == 0) + rc = system(DMIDECODE_CMD" -t 4 | /usr/bin/grep " + "'Ampere(R) Altra(R)' > /dev/null"); + else + rc = system("/usr/bin/lscpu | /usr/bin/grep " + "'Ampere(R) Altra(R)' > /dev/null"); + if (rc == -1 || !WIFEXITED(rc) || WEXITSTATUS(rc)) + return; + + ampere_ipmitool = true; +} + +static void ras_report_aer_ipmi(int severity_val, struct ras_aer_event *ev) +{ + char ipmi_add_sel[114]; + uint8_t sel_data[5]; + int seg, bus, dev, fn, rc; + + if (!ampere_ipmitool) + return; + + /* + * Get PCIe AER error source seg/bus/dev/fn and save it into + * BMC OEM SEL, ipmitool raw 0x0a 0x44 is IPMI command-Add SEL + * entry, please refer IPMI specification chapter 31.6. 0xcd3a + * is manufactuer ID(ampere),byte 12 is sensor num(CE is 0xBF, + * UE is 0xCA), byte 13~14 is segment number, byte 15 is bus + * number, byte 16[7:3] is device number, byte 16[2:0] is + * function number. + */ + + switch (severity_val) { + case HW_EVENT_AER_UNCORRECTED_NON_FATAL: + case HW_EVENT_AER_UNCORRECTED_FATAL: + sel_data[0] = 0xca; + break; + case HW_EVENT_AER_CORRECTED: + default: + sel_data[0] = 0xbf; + } + + sscanf(ev->dev_name, "%x:%x:%x.%x", &seg, &bus, &dev, &fn); + + sel_data[1] = seg & 0xff; + sel_data[2] = (seg & 0xff00) >> 8; + sel_data[3] = bus; + sel_data[4] = (((dev & 0x1f) << 3) | (fn & 0x7)); + + sprintf(ipmi_add_sel, IPMITOOL_CMD + " raw 0x0a 0x44 0x00 0x00 0xc0 0x00 0x00 0x00 0x00 0x3a 0xcd 0x00 0xc0 0x%02x 0x%02x 0x%02x 0x%02x 0x%02x", + sel_data[0], sel_data[1], sel_data[2], sel_data[3], sel_data[4]); + + rc = system(ipmi_add_sel); + if (rc == -1 || !WIFEXITED(rc) || WEXITSTATUS(rc)) + log(TERM, LOG_ERR, "ipmitool command failed [%d]", rc); +} +#endif + #define BUF_LEN 1024 int ras_aer_event_handler(struct trace_seq *s, @@ -67,9 +152,6 @@ int ras_aer_event_handler(struct trace_seq *s, struct tm *tm; struct ras_aer_event ev; char buf[BUF_LEN]; - char ipmi_add_sel[105]; - uint8_t sel_data[5]; - int seg, bus, dev, fn; /* * Newer kernels (3.10-rc1 or upper) provide an uptime clock. @@ -132,24 +214,20 @@ int ras_aer_event_handler(struct trace_seq *s, switch (severity_val) { case HW_EVENT_AER_UNCORRECTED_NON_FATAL: ev.error_type = "Uncorrected (Non-Fatal)"; - sel_data[0] = 0xca; break; case HW_EVENT_AER_UNCORRECTED_FATAL: ev.error_type = "Uncorrected (Fatal)"; - sel_data[0] = 0xca; break; case HW_EVENT_AER_CORRECTED: ev.error_type = "Corrected"; - sel_data[0] = 0xbf; break; default: ev.error_type = "Unknown severity"; - sel_data[0] = 0xbf; } trace_seq_puts(s, ev.error_type); - /* Insert data into the SGBD */ #ifdef HAVE_SQLITE3 + /* Insert data into the SGBD */ ras_store_aer_event(ras, &ev); #endif @@ -159,28 +237,16 @@ int ras_aer_event_handler(struct trace_seq *s, #endif #ifdef HAVE_AMP_NS_DECODE - /* - * Get PCIe AER error source seg/bus/dev/fn and save it into - * BMC OEM SEL, ipmitool raw 0x0a 0x44 is IPMI command-Add SEL - * entry, please refer IPMI specificaiton chapter 31.6. 0xcd3a - * is manufactuer ID(ampere),byte 12 is sensor num(CE is 0xBF, - * UE is 0xCA), byte 13~14 is segment number, byte 15 is bus - * number, byte 16[7:3] is device number, byte 16[2:0] is - * function number - */ - sscanf(ev.dev_name, "%x:%x:%x.%x", &seg, &bus, &dev, &fn); - - sel_data[1] = seg & 0xff; - sel_data[2] = (seg & 0xff00) >> 8; - sel_data[3] = bus; - sel_data[4] = (((dev & 0x1f) << 3) | (fn & 0x7)); - - sprintf(ipmi_add_sel, - "ipmitool raw 0x0a 0x44 0x00 0x00 0xc0 0x00 0x00 0x00 0x00 0x3a 0xcd 0x00 0xc0 0x%02x 0x%02x 0x%02x 0x%02x 0x%02x", - sel_data[0], sel_data[1], sel_data[2], sel_data[3], sel_data[4]); - - system(ipmi_add_sel); + /* Give a chance to provide AER error though IPMI */ + ras_report_aer_ipmi(severity_val, &ev); #endif return 0; } + +void ras_aer_handler_init(void) +{ +#ifdef HAVE_AMP_NS_DECODE + ras_report_aer_ipmi_init(); +#endif +} diff --git a/ras-aer-handler.h b/ras-aer-handler.h index 876a60ba..a1cd16a1 100644 --- a/ras-aer-handler.h +++ b/ras-aer-handler.h @@ -26,4 +26,5 @@ int ras_aer_event_handler(struct trace_seq *s, struct pevent_record *record, struct event_format *event, void *context); +void ras_aer_handler_init(void); #endif diff --git a/ras-events.c b/ras-events.c index fe4bd26a..633a9713 100644 --- a/ras-events.c +++ b/ras-events.c @@ -824,6 +824,7 @@ int handle_ras_events(int record_events) "ras", "mc_event"); #ifdef HAVE_AER + ras_aer_handler_init(); rc = add_event_handler(ras, pevent, page_size, "ras", "aer_event", ras_aer_event_handler, NULL, AER_EVENT); if (!rc) From f3f4d9c3f52ffa26665b173b22140a2fb33b7913 Mon Sep 17 00:00:00 2001 From: William Roche Date: Mon, 6 Dec 2021 13:55:55 +0100 Subject: [PATCH 02/61] Enrich ras_report_aer_ipmi_init() comments. Take Jason Tian's feedback into account to provide more details about the conditions allowing the use of ipmitool on Altra or Altra Max platforms. Signed-off-by: William Roche --- ras-aer-handler.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ras-aer-handler.c b/ras-aer-handler.c index 24872f17..8f2cb11a 100644 --- a/ras-aer-handler.c +++ b/ras-aer-handler.c @@ -71,6 +71,9 @@ static void ras_report_aer_ipmi_init(void) /* * Verify on startup if we are on an Ampere Altra or Altra Max * platform, to set the use of ipmitool (if installed). + * This depends on BIOS implementation to provide the CPU information. + * If the BIOS doesn't provide it or gives a different string, the + * ipmitool use will be disabled. */ if (stat(IPMITOOL_CMD, &st) != 0) return; @@ -78,7 +81,7 @@ static void ras_report_aer_ipmi_init(void) if ((uname(&unm) != 0) || (strncmp(unm.machine, "aarch64", 8) != 0)) return; - /* prefer dmidecode as lscpu may not have the necessary dmi info */ + /* prefer dmidecode (if installed) as only lscpu newer than 2.37 gets dmi info */ if (stat(DMIDECODE_CMD, &st) == 0) rc = system(DMIDECODE_CMD" -t 4 | /usr/bin/grep " "'Ampere(R) Altra(R)' > /dev/null"); From b14b90178a8cda2c01aba726f1b26eeece0c001b Mon Sep 17 00:00:00 2001 From: Stijn Tintel Date: Wed, 1 Sep 2021 03:32:18 +0300 Subject: [PATCH 03/61] rasdaemon: fix compile against musl libc Fix the following compile errors that occurs when building against musl: ras-events.c: In function 'read_ras_event_all_cpus': ras-events.c:366:16: error: 'PATH_MAX' undeclared (first use in this function) 366 | char pipe_raw[PATH_MAX]; | ^~~~~~~~ ras-events.c: In function 'handle_ras_events_cpu': ras-events.c:564:16: error: 'PATH_MAX' undeclared (first use in this function) 564 | char pipe_raw[PATH_MAX]; | Signed-off-by: Stijn Tintel Signed-off-by: Mauro Carvalho Chehab --- ras-events.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ras-events.c b/ras-events.c index 633a9713..48b03b44 100644 --- a/ras-events.c +++ b/ras-events.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include From fc3f8de356d9f6ae5c3f05f30258b21851033eb0 Mon Sep 17 00:00:00 2001 From: Muralidhara M K Date: Tue, 27 Jul 2021 06:36:45 -0500 Subject: [PATCH 04/61] rasdaemon: ras-mc-ctl: Fix script to parse dimm sizes Removes trailing spaces at the end of a line from file location and fixes --layout option to parse dimm nodes to get the size of each dimm from ras-mc-ctl. Issue is reported https://github.com/mchehab/rasdaemon/issues/43 Where '> ras-mc-ctl --layout' reports all 0s With this change the layout option prints the correct dimm sizes > sudo ras-mc-ctl --layout +-----------------------------------------------+ | mc0 | | csrow0 | csrow1 | csrow2 | csrow3 | ----------+-----------------------------------------------+ ... channel7: | 16384 MB | 0 MB | 0 MB | 0 MB | channel6: | 16384 MB | 0 MB | 0 MB | 0 MB | ... ----------+-----------------------------------------------+ Signed-off-by: Muralidhara M K Signed-off-by: Naveen Krishna Chatradhi Cc: Yazen Ghannam Signed-off-by: Mauro Carvalho Chehab Link: https://lkml.kernel.org/r/20210810183855.129076-1-nchatrad@amd.com/ --- util/ras-mc-ctl.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/ras-mc-ctl.in b/util/ras-mc-ctl.in index 1e3aeb7e..b22dd60b 100755 --- a/util/ras-mc-ctl.in +++ b/util/ras-mc-ctl.in @@ -246,6 +246,7 @@ sub parse_dimm_nodes if (($file =~ /max_location$/)) { open IN, $file; my $location = ; + $location =~ s/\s+$//; close IN; my @temp = split(/ /, $location); @@ -288,6 +289,7 @@ sub parse_dimm_nodes open IN, $file; my $location = ; + $location =~ s/\s+$//; close IN; my @pos; From c66be8e983f1b6ab4275c0b03e5083f611a88c6b Mon Sep 17 00:00:00 2001 From: Justin Vreeland Date: Tue, 2 Nov 2021 19:51:50 -0700 Subject: [PATCH 05/61] Update ras-mc-ctl manpage to match current options Signed-off-by: Justin Vreeland Signed-off-by: Mauro Carvalho Chehab --- man/ras-mc-ctl.8.in | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/man/ras-mc-ctl.8.in b/man/ras-mc-ctl.8.in index 26230e0a..a605122f 100644 --- a/man/ras-mc-ctl.8.in +++ b/man/ras-mc-ctl.8.in @@ -79,9 +79,27 @@ Specify an alternate location for the labels database. Specify a delay of \fBtime\fR seconds before registering DIMM labels. Only meaninful if used together with --register-labels. .TP -.BI "--layout +.BI "--layout" Prints the memory layout as detected by the EDAC driver. Useful to check if the EDAC driver is properly detecting the memory controller architecture. +.TP +.BI "--summary" +Presents a summary of the logged errors. +.TP +.BI "--errors" +Shows the errors stored at the error database. +.TP +.BI "--error-count" +Shows the corrected and uncorrected error counts using sysfs. +.TP +.BI "--vendor-errors-summary="platform-id +Pressents a summary of the vendor-specific logged errors. +.TP +.BI "--vendor-errors="platform-id +Shows the vendor-specific errors stored in the error database. +.TP +.BI "--vendor-platforms" +Shows the supported platforms with platform-ids for the vendor-specific errors. .SH MAINBOARD CONFIGURATION .PP From ae89390c36e596e88b8043cac74a97b6f3e47525 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Tue, 7 Dec 2021 17:57:08 +0700 Subject: [PATCH 06/61] add labels for asrock x570 motherboard Signed-off-by: Steven Johnson Signed-off-by: Mauro Carvalho Chehab --- labels/asrock | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 labels/asrock diff --git a/labels/asrock b/labels/asrock new file mode 100644 index 00000000..6be70d39 --- /dev/null +++ b/labels/asrock @@ -0,0 +1,20 @@ +# RASDAEMON Motherboard DIMM labels Database file. +# +# Vendor-name and model-name are found from the program 'dmidecode' +# labels are found from the silk screen on the motherboard. +# +#Vendor: +# Product: +# Model: +#