From 4a5db2bbc071289cea29204d12d17f0f30c9bf99 Mon Sep 17 00:00:00 2001 From: Ruidong Tian Date: Fri, 11 Apr 2025 13:30:10 +0800 Subject: [PATCH] rasdaemon: support nvgpu event Use nvml library to report nvgpu event. New environment NVGPU_DISABLE_EVENT indicate registered events. Signed-off-by: Ruidong Tian --- .gitignore | 1 + Makefile.am | 12 +++- configure.ac | 11 +++ contrib/nvml.py | 77 +++++++++++++++++++++ misc/rasdaemon.env | 7 ++ ras-nvgpu-nvml.c | 165 +++++++++++++++++++++++++++++++++++++++++++++ ras-nvgpu.c | 54 +++++++++++++++ ras-nvgpu.h | 14 ++++ rasdaemon.c | 27 ++++++++ 9 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 contrib/nvml.py create mode 100644 ras-nvgpu-nvml.c create mode 100644 ras-nvgpu.c create mode 100644 ras-nvgpu.h diff --git a/.gitignore b/.gitignore index 8cd73822..a3bedf94 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ rasdaemon *.h~ rasdaemon-*.tar.bz2 rasdaemon-*.src.rpm +ras-nvgpu-nvml.h diff --git a/Makefile.am b/Makefile.am index 01132fec..838eba10 100644 --- a/Makefile.am +++ b/Makefile.am @@ -7,10 +7,12 @@ SYSTEMD_SERVICES_IN = misc/rasdaemon.service.in misc/ras-mc-ctl.service.in SYSTEMD_SERVICES = $(SYSTEMD_SERVICES_IN:.service.in=.service) EXTRA_DIST = \ $(SYSTEMD_SERVICES_IN) misc/rasdaemon.env \ + contrib/nvml.py \ contrib/mc_event_trigger \ contrib/mem_fail_trigger CLEANFILES= \ + ras-nvgpu-nvml.h \ misc/ras-mc-ctl.service \ misc/rasdaemon.service @@ -90,8 +92,14 @@ endif if WITH_JAGUAR_NS_DECODE rasdaemon_SOURCES += non-standard-jaguarmicro.c endif +if WITH_NVGPU + BUILT_SOURCES = ras-nvgpu-nvml.h +ras-nvgpu-nvml.h: contrib/nvml.py + python3 $< > $@ + rasdaemon_SOURCES += ras-nvgpu.c ras-nvgpu-nvml.c +endif -rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS) +rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS) -ldl rasdaemon_CFLAGS = $(SQLITE3_CFLAGS) $(LIBTRACEEVENT_CFLAGS) include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \ @@ -100,7 +108,7 @@ include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \ ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h \ non-standard-hisilicon.h non-standard-ampere.h ras-memory-failure-handler.h \ ras-cxl-handler.h ras-cpu-isolation.h queue.h non-standard-yitian.h \ - non-standard-jaguarmicro.h trigger.h unified-sel.h + non-standard-jaguarmicro.h trigger.h unified-sel.h ras-nvgpu.h # This rule can't be called with more than one Makefile job (like make -j8) # I can't figure out a way to fix that diff --git a/configure.ac b/configure.ac index 1cb00b6d..47e7551d 100644 --- a/configure.ac +++ b/configure.ac @@ -244,6 +244,16 @@ AS_IF([test "x$enable_yitian_ns_decode" = "xyes" || test "x$enable_all" == "xyes AM_CONDITIONAL([WITH_YITIAN_NS_DECODE], [test x$enable_yitian_ns_decode = xyes || test x$enable_all == xyes]) AM_COND_IF([WITH_YITIAN_NS_DECODE], [USE_YITIAN_NS_DECODE="yes"], [USE_YITIAN_NS_DECODE="no"]) +AC_ARG_ENABLE([nvgpu], + AS_HELP_STRING([--enable-nvgpu], [enable NVGPU events])) + +AS_IF([test "x$enable_nvgpu" = "xyes" || test "x$enable_all" == "xyes"], [ + AC_DEFINE(HAVE_NVGPU,1,"have NVGPU events collect") + AC_SUBST([WITH_NVGPU]) +]) +AM_CONDITIONAL([WITH_NVGPU], [test x$enable_nvgpu = xyes || test x$enable_all == xyes]) +AM_COND_IF([WITH_NVGPU], [USE_NVGPU="yes"], [USE_NVGPU="no"]) + test "$sysconfdir" = '${prefix}/etc' && sysconfdir=/etc CFLAGS="$CFLAGS -Wall -Wmissing-prototypes -Wstrict-prototypes" @@ -290,4 +300,5 @@ compile time options summary CPU fault isolation : $USE_CPU_FAULT_ISOLATION YITIAN RAS errors : $USE_YITIAN_NS_DECODE JAGUAR RAS errors : $USE_JAGUAR_NS_DECODE + NVGPU RAS errors : $USE_NVGPU EOF diff --git a/contrib/nvml.py b/contrib/nvml.py new file mode 100644 index 00000000..9f2c57d0 --- /dev/null +++ b/contrib/nvml.py @@ -0,0 +1,77 @@ +import re + +PATH="/usr/local/cuda/include/nvml.h" +func = ["nvmlInit", + "nvmlDeviceGetSupportedEventTypes", + "nvmlDeviceRegisterEvents", + "nvmlEventSetCreate", + "nvmlEventSetWait", + "nvmlDeviceGetCount", + "nvmlDeviceGetHandleByIndex", + "nvmlDeviceGetPciInfo", + "nvmlEventSetFree", + "nvmlShutdown"] + +pattern = re.compile( + r'^nvmlReturn_t DECLDIR\s+({})(\(.*?\));'.format('|'.join(map(re.escape, func))), + flags=re.MULTILINE +) + +type_pattern = re.compile( + r'^#define\s+nvmlEventType(\w+)\s+0x.*', + flags=re.MULTILINE +) + +with open(PATH, 'r') as file: + content = file.read() + matched_lines = pattern.findall(content) + type_lines = type_pattern.findall(content) + +func_declares = [] +func_defs = [] +func_inits = [] +type_strs = [] + +for match in matched_lines: + func_declares.append('typedef nvmlReturn_t (*my_{}_p){};'.format(match[0], match[1])) + func_defs.append('my_{}_p my_{};'.format(match[0], match[0])) + func_inits.append('my_{0} = (my_{0}_p)dlsym(handle, "{0}"); \ + \n\tif (!my_{0}) {{ \ + \n\t\tprintf(\"Failed to load {0}: %s\\n\", dlerror()); \ + \n\t\treturn -1; \ + \n\t}}'.format(match[0])) + +for type_line in type_lines: + type_strs.append('case nvmlEventType{}: return \"{}\";'.format(type_line, type_line)) + +print(''' +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* + * Copyright (C) 2025 Alibaba Inc + */ + +''' +) +print('#include \ + \n#include \ + \n#include "/usr/local/cuda/include/nvml.h"') +print('\ntypedef const char* (*my_nvmlErrorString_p)(nvmlReturn_t result);') +print('\n'.join(func_declares)) +print('\nmy_nvmlErrorString_p my_nvmlErrorString;') +print('\n'.join(func_defs)) +print('\nstatic int my_nvml_setup(void* handle) \n{{\n\t{}{}\n\treturn 0;\n}}'.format('\n\t'.join(func_inits), + '\n\tmy_nvmlErrorString = (my_nvmlErrorString_p)dlsym(handle, "nvmlErrorString"); \ + \n\tif (!my_nvmlErrorString) { \ + \n\t\tprintf(\"Failed to load nvmlErrorString: %s\\n\", dlerror()); \ + \n\t\treturn -1; \ + \n\t}')) +print('\nstatic const char* my_nvmlEventTypeString(unsigned long long type) \n{{ \ + \n\n\tswitch (type) {{ \ + \n\t{} \ + \n\tdefault: return \"Unknown\"; \ + \n\t}} \ + \n\treturn \"Unknown\"; \ + \n}}'.format('\n\t'.join(type_strs))) + + diff --git a/misc/rasdaemon.env b/misc/rasdaemon.env index 963aaa05..97ac2dc9 100644 --- a/misc/rasdaemon.env +++ b/misc/rasdaemon.env @@ -88,3 +88,10 @@ TRIGGER_DIR= # MC_UE_TRIGGER=mc_event_trigger MC_CE_TRIGGER= MC_UE_TRIGGER= + +# Registered event type for nvgpu, default is +# nvmlEventTypeAll & ~nvmlEventTypeClock +# ref: https://docs.nvidia.com/deploy/nvml-api/group__nvmlEventType.html +# For example: +# NVGPU_DISABLE_EVENT="0x10" # disable nvmlEventTypeClock +NVGPU_DISABLE_EVENT="0x10" diff --git a/ras-nvgpu-nvml.c b/ras-nvgpu-nvml.c new file mode 100644 index 00000000..b7057feb --- /dev/null +++ b/ras-nvgpu-nvml.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* + * Copyright (C) 2025 Alibaba Inc + */ + +#include +#include + +#include "ras-logger.h" +#include "ras-nvgpu-nvml.h" +#include "ras-nvgpu.h" +#include "trace-seq.h" +#include "types.h" + +#define XID_EVENT_NAME "xid" + +const char *lib_name[] = { + "/lib64/libnvidia-ml.so", + "/lib64/libnvidia-ml.so.1", + "/usr/local/cuda/targets/x86_64-linux/lib/stubs/libnvidia-ml.so", + "/usr/local/cuda/targets/sbsa-linux/lib/stubs/libnvidia-ml.so" +}; + +static void *find_lib(void) +{ + void *handle = NULL; + + for (int i = 0; i < ARRAY_SIZE(lib_name); i++) { + handle = dlopen(lib_name[i], RTLD_LAZY); + if (handle) + return handle; + } + + log(ALL, LOG_ERR, "Failed to load libnvidia-ml\n"); + return NULL; +} + +static int report_ras_gpu_nvml(nvmlEventData_t *data, nvmlDevice_t *devices) +{ + struct trace_seq s; + nvmlPciInfo_t pci; + + my_nvmlDeviceGetPciInfo(data->device, &pci); + + trace_seq_init(&s); + if (data->eventType == nvmlEventTypeXidCriticalError) { + trace_seq_printf(&s, "%16s-%-10d [%03d] %s %6.6f %25s: ", + "<...>", 0, -1, "....", 0.0f, XID_EVENT_NAME); + trace_seq_printf(&s, "xid: %lld ", data->eventData); + } else { + trace_seq_printf(&s, "%16s-%-10d [%03d] %s %6.6f %25s: ", + "<...>", 0, -1, "....", 0.0f, NVGPU_EVENT_NAME); + trace_seq_printf(&s, "event_type: %s ", my_nvmlEventTypeString(data->eventType)); + trace_seq_printf(&s, "data: %lld ", data->eventData); + } + + trace_seq_printf(&s, "pci_port: " NVML_DEVICE_PCI_BUS_ID_FMT " ", NVML_DEVICE_PCI_BUS_ID_FMT_ARGS(&pci)); + trace_seq_printf(&s, "gpu-i: %x ", data->gpuInstanceId); + trace_seq_printf(&s, "gpu-ci: %x ", data->computeInstanceId); + + trace_seq_terminate(&s); + trace_seq_do_printf(&s); + printf("\n"); + fflush(stdout); + trace_seq_destroy(&s); + + return 0; +} + +int ras_nvgpu_nvml_handle(void) +{ + void *nvml_handle; + nvmlReturn_t ret; + unsigned int device_count; + nvmlDevice_t *devices; + nvmlEventSet_t event_set; + char *event_types_str = NULL; + unsigned long long disable = 0, event_types = 0; + nvmlEventData_t event_data; + + nvml_handle = find_lib(); + if (!nvml_handle) { + log(ALL, LOG_ERR, "Failed to load libnvidia-ml: %s\n", dlerror()); + return 1; + } + + if (my_nvml_setup(nvml_handle)) { + log(ALL, LOG_ERR, "Failed to setup libnvidia-ml\n"); + dlclose(nvml_handle); + return 1; + } + + ret = my_nvmlInit(); + if (ret) { + log(ALL, LOG_ERR, "NVML Init failed: %s\n", my_nvmlErrorString(ret)); + goto free_dl; + } + + ret = my_nvmlDeviceGetCount(&device_count); + if (ret) { + log(ALL, LOG_ERR, "Get device count failed: %s\n", my_nvmlErrorString(ret)); + goto free_nvml; + } + + devices = malloc(device_count * sizeof(nvmlDevice_t)); + if (!devices) { + log(ALL, LOG_ERR, "Failed to allocate memory for devices\n"); + goto free_nvml; + } + + for (unsigned int i = 0; i < device_count; i++) { + ret = my_nvmlDeviceGetHandleByIndex(i, &devices[i]); + if (ret) { + log(ALL, LOG_ERR, "Get device handle failed: %s\n", my_nvmlErrorString(ret)); + goto free_dev; + } + } + + ret = my_nvmlEventSetCreate(&event_set); + if (ret) { + log(ALL, LOG_ERR, "Create event set failed: %s\n", my_nvmlErrorString(ret)); + goto free_dev; + } + + event_types_str = getenv("NVGPU_DISABLE_EVENT"); + if (event_types_str) { + disable = strtoull(event_types_str, NULL, 0); + log(ALL, LOG_INFO, "Disable NVGPU events %s\n", my_nvmlEventTypeString(disable)); + } + + for (unsigned int i = 0; i < device_count; i++) { + ret = my_nvmlDeviceGetSupportedEventTypes(devices[i], &event_types); + if (ret) { + log(ALL, LOG_ERR, "Get support events failed: %s\n", my_nvmlErrorString(ret)); + goto free_event; + } + + ret = my_nvmlDeviceRegisterEvents(devices[i], event_types & ~disable, event_set); + if (ret) { + log(ALL, LOG_ERR, "Register events failed: %s\n", my_nvmlErrorString(ret)); + goto free_event; + } + } + + while (1) { + ret = my_nvmlEventSetWait(event_set, &event_data, -1); + if (!ret) + report_ras_gpu_nvml(&event_data, devices); + else { + log(ALL, LOG_ERR, "Wait for event failed: %s\n", my_nvmlErrorString(ret)); + break; + } + } + +free_event: + my_nvmlEventSetFree(event_set); +free_dev: + free(devices); +free_nvml: + my_nvmlShutdown(); +free_dl: + dlclose(nvml_handle); + return ret; +} diff --git a/ras-nvgpu.c b/ras-nvgpu.c new file mode 100644 index 00000000..5c632791 --- /dev/null +++ b/ras-nvgpu.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* + * Copyright (C) 2025 Alibaba Inc + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ras-events.h" +#include "ras-logger.h" +#include "ras-nvgpu.h" +void *ras_nvgpu_handle(void *arg) +{ + (void)arg; + sigset_t set; + struct stat st; + int retry = 3; + + if (stat("/dev/nvidia0", &st) == -1) { + log(ALL, LOG_WARNING, "NVIDIA device not found: %s\n", strerror(errno)); + return NULL; + } + if (!S_ISCHR(st.st_mode)) { + log(ALL, LOG_WARNING, "NVIDIA device is not a character device\n"); + return NULL; + } + + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGQUIT); + if (pthread_sigmask(SIG_BLOCK, &set, NULL) != 0) { + log(ALL, LOG_ERR, "Failed to set thread signal mask\n"); + return NULL; + } + + while (retry--) { + if (ras_nvgpu_nvml_handle()) { + log(ALL, LOG_ERR, "NVGPU handle retry %d\n", retry); + sleep(10); + } + } + + log(ALL, LOG_ERR, "NVGPU handle fail, exit from nvgpu thread\n"); + + return NULL; +} diff --git a/ras-nvgpu.h b/ras-nvgpu.h new file mode 100644 index 00000000..32827ad3 --- /dev/null +++ b/ras-nvgpu.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* + * Copyright (C) 2025 Alibaba Inc + */ + +#ifndef __RAS_NVGPU_H +#define __RAS_NVGPU_H + +#define NVGPU_EVENT_NAME "nvgpu" + +void *ras_nvgpu_handle(void *arg); +int ras_nvgpu_nvml_handle(void); +#endif diff --git a/rasdaemon.c b/rasdaemon.c index 840be61b..a33ff7fa 100644 --- a/rasdaemon.c +++ b/rasdaemon.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -13,6 +14,7 @@ #include "ras-events.h" #include "ras-logger.h" #include "ras-record.h" +#include "ras-nvgpu.h" #include "types.h" /* @@ -209,7 +211,32 @@ int main(int argc, char *argv[]) if (daemon(0, 0)) exit(EXIT_FAILURE); +#ifdef HAVE_NVGPU + pthread_t nvgpu_thread = 0, main_thread = pthread_self(); + bool nvgpu_enable = true; + + if (choices_disable && strlen(choices_disable) != 0 && + strstr(choices_disable, NVGPU_EVENT_NAME)) { + nvgpu_enable = false; + log(ALL, LOG_INFO, "Disable nvgpu event.\n"); + } + + if (nvgpu_enable) { + if (pthread_create(&nvgpu_thread, NULL, ras_nvgpu_handle, &main_thread) != 0) { + log(ALL, LOG_ERR, "Failed to create XID thread\n"); + pthread_cancel(nvgpu_thread); + exit(EXIT_FAILURE); + } + pthread_detach(nvgpu_thread); + log(ALL, LOG_INFO, "Create pthread to handle NVGPU events.\n"); + } +#endif handle_ras_events(args.record_events, args.enable_ipmitool); +#ifdef HAVE_NVGPU + if (nvgpu_enable) + pthread_cancel(nvgpu_thread); +#endif + return 0; }