support nvgpu event

.gitignore

@@ -28,3 +28,4 @@ rasdaemon
 *.h~
 rasdaemon-*.tar.bz2
 rasdaemon-*.src.rpm
+ras-nvgpu-nvml.h

Makefile.am

@@ -7,10 +7,12 @@ SYSTEMD_SERVICES_IN = misc/rasdaemon.service.in misc/ras-mc-ctl.service.in
 SYSTEMD_SERVICES = $(SYSTEMD_SERVICES_IN:.service.in=.service)
 EXTRA_DIST = \
 	$(SYSTEMD_SERVICES_IN) misc/rasdaemon.env \
+	contrib/nvml.py \
 	contrib/mc_event_trigger \
 	contrib/mem_fail_trigger
 
 CLEANFILES= \
+	ras-nvgpu-nvml.h \
 	misc/ras-mc-ctl.service	\
 	misc/rasdaemon.service
 
@@ -90,8 +92,14 @@ endif
 if WITH_JAGUAR_NS_DECODE
    rasdaemon_SOURCES += non-standard-jaguarmicro.c
 endif
+if WITH_NVGPU
+   BUILT_SOURCES = ras-nvgpu-nvml.h
+ras-nvgpu-nvml.h: contrib/nvml.py
+	python3 $< > $@
+   rasdaemon_SOURCES += ras-nvgpu.c ras-nvgpu-nvml.c
+endif
 
-rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS)
+rasdaemon_LDADD = -lpthread $(SQLITE3_LIBS) $(LIBTRACEEVENT_LIBS) -ldl
 rasdaemon_CFLAGS = $(SQLITE3_CFLAGS) $(LIBTRACEEVENT_CFLAGS)
 
 include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \
@@ -100,7 +108,7 @@ include_HEADERS = config.h types.h ras-events.h ras-logger.h ras-mc-handler.h \
 		  ras-devlink-handler.h ras-diskerror-handler.h rbtree.h ras-page-isolation.h \
 		  non-standard-hisilicon.h non-standard-ampere.h ras-memory-failure-handler.h \
 		  ras-cxl-handler.h ras-cpu-isolation.h queue.h non-standard-yitian.h \
-		  non-standard-jaguarmicro.h trigger.h unified-sel.h
+		  non-standard-jaguarmicro.h trigger.h unified-sel.h ras-nvgpu.h
 
 # This rule can't be called with more than one Makefile job (like make -j8)
 # I can't figure out a way to fix that

configure.ac

@@ -244,6 +244,16 @@ AS_IF([test "x$enable_yitian_ns_decode" = "xyes" || test "x$enable_all" == "xyes
 AM_CONDITIONAL([WITH_YITIAN_NS_DECODE], [test x$enable_yitian_ns_decode = xyes || test x$enable_all == xyes])
 AM_COND_IF([WITH_YITIAN_NS_DECODE], [USE_YITIAN_NS_DECODE="yes"], [USE_YITIAN_NS_DECODE="no"])
 
+AC_ARG_ENABLE([nvgpu],
+    AS_HELP_STRING([--enable-nvgpu], [enable NVGPU events]))
+
+AS_IF([test "x$enable_nvgpu" = "xyes" || test "x$enable_all" == "xyes"], [
+  AC_DEFINE(HAVE_NVGPU,1,"have NVGPU events collect")
+  AC_SUBST([WITH_NVGPU])
+])
+AM_CONDITIONAL([WITH_NVGPU], [test x$enable_nvgpu = xyes || test x$enable_all == xyes])
+AM_COND_IF([WITH_NVGPU], [USE_NVGPU="yes"], [USE_NVGPU="no"])
+
 test "$sysconfdir" = '${prefix}/etc' && sysconfdir=/etc
 
 CFLAGS="$CFLAGS -Wall -Wmissing-prototypes -Wstrict-prototypes"
@@ -290,4 +300,5 @@ compile time options summary
     CPU fault isolation : $USE_CPU_FAULT_ISOLATION
     YITIAN RAS errors   : $USE_YITIAN_NS_DECODE
     JAGUAR RAS errors   : $USE_JAGUAR_NS_DECODE
+    NVGPU RAS errors    : $USE_NVGPU
 EOF

contrib/nvml.py

@@ -0,0 +1,77 @@
+import re
+
+PATH="/usr/local/cuda/include/nvml.h"
+func = ["nvmlInit",
+        "nvmlDeviceGetSupportedEventTypes",
+        "nvmlDeviceRegisterEvents",
+        "nvmlEventSetCreate",
+        "nvmlEventSetWait",
+        "nvmlDeviceGetCount",
+        "nvmlDeviceGetHandleByIndex",
+        "nvmlDeviceGetPciInfo",
+        "nvmlEventSetFree",
+        "nvmlShutdown"]
+
+pattern = re.compile(
+        r'^nvmlReturn_t DECLDIR\s+({})(\(.*?\));'.format('|'.join(map(re.escape, func))),
+        flags=re.MULTILINE
+)
+
+type_pattern = re.compile(
+        r'^#define\s+nvmlEventType(\w+)\s+0x.*',
+        flags=re.MULTILINE
+)
+
+with open(PATH, 'r') as file:
+        content = file.read()
+        matched_lines = pattern.findall(content)
+        type_lines = type_pattern.findall(content)
+
+func_declares = []
+func_defs = []
+func_inits = []
+type_strs = []
+
+for match in matched_lines:
+        func_declares.append('typedef nvmlReturn_t (*my_{}_p){};'.format(match[0], match[1]))
+        func_defs.append('my_{}_p my_{};'.format(match[0], match[0]))
+        func_inits.append('my_{0} = (my_{0}_p)dlsym(handle, "{0}"); \
+                                \n\tif (!my_{0}) {{ \
+                                \n\t\tprintf(\"Failed to load {0}: %s\\n\", dlerror()); \
+                                \n\t\treturn -1; \
+                                \n\t}}'.format(match[0]))
+
+for type_line in type_lines:
+        type_strs.append('case nvmlEventType{}: return \"{}\";'.format(type_line, type_line))
+
+print('''
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/*
+ * Copyright (C) 2025 Alibaba Inc
+ */
+
+'''
+)
+print('#include <dlfcn.h>\
+        \n#include <stdio.h>\
+        \n#include "/usr/local/cuda/include/nvml.h"')
+print('\ntypedef const char* (*my_nvmlErrorString_p)(nvmlReturn_t result);')
+print('\n'.join(func_declares))
+print('\nmy_nvmlErrorString_p my_nvmlErrorString;')
+print('\n'.join(func_defs))
+print('\nstatic int my_nvml_setup(void* handle) \n{{\n\t{}{}\n\treturn 0;\n}}'.format('\n\t'.join(func_inits),
+        '\n\tmy_nvmlErrorString = (my_nvmlErrorString_p)dlsym(handle, "nvmlErrorString"); \
+         \n\tif (!my_nvmlErrorString) { \
+         \n\t\tprintf(\"Failed to load nvmlErrorString: %s\\n\", dlerror()); \
+         \n\t\treturn -1; \
+         \n\t}'))
+print('\nstatic const char* my_nvmlEventTypeString(unsigned long long type) \n{{ \
+        \n\n\tswitch (type) {{ \
+        \n\t{} \
+        \n\tdefault: return \"Unknown\"; \
+        \n\t}} \
+        \n\treturn \"Unknown\"; \
+        \n}}'.format('\n\t'.join(type_strs)))
+
+

misc/rasdaemon.env

@@ -88,3 +88,10 @@ TRIGGER_DIR=
 #   MC_UE_TRIGGER=mc_event_trigger
 MC_CE_TRIGGER=
 MC_UE_TRIGGER=
+
+# Registered event type for nvgpu, default is
+# nvmlEventTypeAll & ~nvmlEventTypeClock
+# ref: https://docs.nvidia.com/deploy/nvml-api/group__nvmlEventType.html
+# For example:
+#   NVGPU_DISABLE_EVENT="0x10" # disable nvmlEventTypeClock
+NVGPU_DISABLE_EVENT="0x10"

ras-nvgpu-nvml.c

@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/*
+ * Copyright (C) 2025 Alibaba Inc
+ */
+
+#include <unistd.h>
+#include <stdlib.h>
+
+#include "ras-logger.h"
+#include "ras-nvgpu-nvml.h"
+#include "ras-nvgpu.h"
+#include "trace-seq.h"
+#include "types.h"
+
+#define XID_EVENT_NAME "xid"
+
+const char *lib_name[] = {
+	"/lib64/libnvidia-ml.so",
+	"/lib64/libnvidia-ml.so.1",
+	"/usr/local/cuda/targets/x86_64-linux/lib/stubs/libnvidia-ml.so",
+	"/usr/local/cuda/targets/sbsa-linux/lib/stubs/libnvidia-ml.so"
+};
+
+static void *find_lib(void)
+{
+	void *handle = NULL;
+
+	for (int i = 0; i < ARRAY_SIZE(lib_name); i++) {
+		handle = dlopen(lib_name[i], RTLD_LAZY);
+		if (handle)
+			return handle;
+	}
+
+	log(ALL, LOG_ERR, "Failed to load libnvidia-ml\n");
+	return NULL;
+}
+
+static int report_ras_gpu_nvml(nvmlEventData_t *data, nvmlDevice_t *devices)
+{
+	struct trace_seq s;
+	nvmlPciInfo_t pci;
+
+	my_nvmlDeviceGetPciInfo(data->device, &pci);
+
+	trace_seq_init(&s);
+	if (data->eventType == nvmlEventTypeXidCriticalError) {
+		trace_seq_printf(&s, "%16s-%-10d [%03d] %s %6.6f %25s: ",
+			"<...>", 0, -1, "....", 0.0f, XID_EVENT_NAME);
+		trace_seq_printf(&s, "xid: %lld ", data->eventData);
+	} else {
+		trace_seq_printf(&s, "%16s-%-10d [%03d] %s %6.6f %25s: ",
+			"<...>", 0, -1, "....", 0.0f, NVGPU_EVENT_NAME);
+		trace_seq_printf(&s, "event_type: %s ", my_nvmlEventTypeString(data->eventType));
+		trace_seq_printf(&s, "data: %lld ", data->eventData);
+	}
+
+	trace_seq_printf(&s, "pci_port: " NVML_DEVICE_PCI_BUS_ID_FMT " ", NVML_DEVICE_PCI_BUS_ID_FMT_ARGS(&pci));
+	trace_seq_printf(&s, "gpu-i: %x ", data->gpuInstanceId);
+	trace_seq_printf(&s, "gpu-ci: %x ", data->computeInstanceId);
+
+	trace_seq_terminate(&s);
+	trace_seq_do_printf(&s);
+	printf("\n");
+	fflush(stdout);
+	trace_seq_destroy(&s);
+
+	return 0;
+}
+
+int ras_nvgpu_nvml_handle(void)
+{
+	void *nvml_handle;
+	nvmlReturn_t ret;
+	unsigned int device_count;
+	nvmlDevice_t *devices;
+	nvmlEventSet_t event_set;
+	char *event_types_str = NULL;
+	unsigned long long disable = 0, event_types = 0;
+	nvmlEventData_t event_data;
+
+	nvml_handle = find_lib();
+	if (!nvml_handle) {
+		log(ALL, LOG_ERR, "Failed to load libnvidia-ml: %s\n", dlerror());
+		return 1;
+	}
+
+	if (my_nvml_setup(nvml_handle)) {
+		log(ALL, LOG_ERR, "Failed to setup libnvidia-ml\n");
+		dlclose(nvml_handle);
+		return 1;
+	}
+
+	ret = my_nvmlInit();
+	if (ret) {
+		log(ALL, LOG_ERR, "NVML Init failed: %s\n", my_nvmlErrorString(ret));
+		goto free_dl;
+	}
+
+	ret = my_nvmlDeviceGetCount(&device_count);
+	if (ret) {
+		log(ALL, LOG_ERR, "Get device count failed: %s\n", my_nvmlErrorString(ret));
+		goto free_nvml;
+	}
+
+	devices = malloc(device_count * sizeof(nvmlDevice_t));
+	if (!devices) {
+		log(ALL, LOG_ERR, "Failed to allocate memory for devices\n");
+		goto free_nvml;
+	}
+
+	for (unsigned int i = 0; i < device_count; i++) {
+		ret = my_nvmlDeviceGetHandleByIndex(i, &devices[i]);
+		if (ret) {
+			log(ALL, LOG_ERR, "Get device handle failed: %s\n", my_nvmlErrorString(ret));
+			goto free_dev;
+		}
+	}
+
+	ret = my_nvmlEventSetCreate(&event_set);
+	if (ret) {
+		log(ALL, LOG_ERR, "Create event set failed: %s\n", my_nvmlErrorString(ret));
+		goto free_dev;
+	}
+
+	event_types_str = getenv("NVGPU_DISABLE_EVENT");
+	if (event_types_str) {
+		disable = strtoull(event_types_str, NULL, 0);
+		log(ALL, LOG_INFO, "Disable NVGPU events %s\n", my_nvmlEventTypeString(disable));
+	}
+
+	for (unsigned int i = 0; i < device_count; i++) {
+		ret = my_nvmlDeviceGetSupportedEventTypes(devices[i], &event_types);
+		if (ret) {
+			log(ALL, LOG_ERR, "Get support events failed: %s\n", my_nvmlErrorString(ret));
+			goto free_event;
+		}
+
+		ret = my_nvmlDeviceRegisterEvents(devices[i], event_types & ~disable, event_set);
+		if (ret) {
+			log(ALL, LOG_ERR, "Register events failed: %s\n", my_nvmlErrorString(ret));
+			goto free_event;
+		}
+	}
+
+	while (1) {
+		ret = my_nvmlEventSetWait(event_set, &event_data, -1);
+		if (!ret)
+			report_ras_gpu_nvml(&event_data, devices);
+		else {
+			log(ALL, LOG_ERR, "Wait for event failed: %s\n", my_nvmlErrorString(ret));
+			break;
+		}
+	}
+
+free_event:
+	my_nvmlEventSetFree(event_set);
+free_dev:
+	free(devices);
+free_nvml:
+	my_nvmlShutdown();
+free_dl:
+	dlclose(nvml_handle);
+	return ret;
+}

ras-nvgpu.c

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+/*
+ * Copyright (C) 2025 Alibaba Inc
+ */
+
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <signal.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "ras-events.h"
+#include "ras-logger.h"
+#include "ras-nvgpu.h"
+void *ras_nvgpu_handle(void *arg)
+{
+	(void)arg;
+	sigset_t set;
+	struct stat st;
+	int retry = 3;
+
+	if (stat("/dev/nvidia0", &st) == -1) {
+		log(ALL, LOG_WARNING, "NVIDIA device not found: %s\n", strerror(errno));
+		return NULL;
+	}
+	if (!S_ISCHR(st.st_mode)) {
+		log(ALL, LOG_WARNING, "NVIDIA device is not a character device\n");
+		return NULL;
+	}
+
+	sigemptyset(&set);
+	sigaddset(&set, SIGINT);
+	sigaddset(&set, SIGTERM);
+	sigaddset(&set, SIGHUP);
+	sigaddset(&set, SIGQUIT);
+	if (pthread_sigmask(SIG_BLOCK, &set, NULL) != 0) {
+		log(ALL, LOG_ERR, "Failed to set thread signal mask\n");
+		return NULL;
+	}
+
+	while (retry--) {
+		if (ras_nvgpu_nvml_handle()) {
+			log(ALL, LOG_ERR, "NVGPU handle retry %d\n", retry);
+			sleep(10);
+		}
+	}
+
+	log(ALL, LOG_ERR, "NVGPU handle fail, exit from nvgpu thread\n");
+
+	return NULL;
+}

ras-nvgpu.h

@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+/*
+ * Copyright (C) 2025 Alibaba Inc
+ */
+
+#ifndef __RAS_NVGPU_H
+#define __RAS_NVGPU_H
+
+#define NVGPU_EVENT_NAME "nvgpu"
+
+void *ras_nvgpu_handle(void *arg);
+int ras_nvgpu_nvml_handle(void);
+#endif