diff --git a/Source/Thunder/CMakeLists.txt b/Source/Thunder/CMakeLists.txt index b2632e9426..88d241a013 100644 --- a/Source/Thunder/CMakeLists.txt +++ b/Source/Thunder/CMakeLists.txt @@ -22,6 +22,27 @@ get_filename_component(TARGET ${CMAKE_CURRENT_SOURCE_DIR} NAME) set(THREADPOOL_COUNT "4" CACHE STRING "The number of threads in the thread pool") set(ENABLE_TRACING_MODULES "" CACHE STRING "A space separated list of specific tracing modules to be enabled at start.") +macro(thunder_postmortem_sink_value VAR_IN VAR_OUT) + string(TOUPPER "${${VAR_IN}}" _s) + if (_s STREQUAL "FILE") + set(${VAR_OUT} 1) + elseif(_s STREQUAL "LOG") + set(${VAR_OUT} 2) + elseif(_s STREQUAL "ALL") + set(${VAR_OUT} 3) + else() + set(${VAR_OUT} 0) + endif() +endmacro() + +set(POSTMORTEM_WORKERPOOL_SINK "FILE" CACHE STRING "Default WorkerPool JSON dump sink on crash (FILE | LOG | ALL | DISABLED)") +set(POSTMORTEM_CALLSTACK_SINK "DISABLED" CACHE STRING "Default inline callstack print sink on crash (FILE | LOG | ALL | DISABLED)") +set_property(CACHE POSTMORTEM_WORKERPOOL_SINK PROPERTY STRINGS FILE LOG ALL DISABLED) +set_property(CACHE POSTMORTEM_CALLSTACK_SINK PROPERTY STRINGS FILE LOG ALL DISABLED) + +thunder_postmortem_sink_value(POSTMORTEM_WORKERPOOL_SINK _wp_sink) +thunder_postmortem_sink_value(POSTMORTEM_CALLSTACK_SINK _cs_sink) + option(SKIP_PERSISTENT_PATH_INSTALL "This will skip installing the persistent path and its privileges. Needed for local build/installs" OFF) add_executable(${TARGET} @@ -38,6 +59,8 @@ target_compile_definitions(${TARGET} NAMESPACE=${NAMESPACE} APPLICATION_NAME=${TARGET} THREADPOOL_COUNT=${THREADPOOL_COUNT} + THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=${_wp_sink} + THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=${_cs_sink} ) target_compile_options (${TARGET} PRIVATE -Wno-psabi) diff --git a/Source/Thunder/Config.h b/Source/Thunder/Config.h index cc2dc43eec..9c69954b41 100644 --- a/Source/Thunder/Config.h +++ b/Source/Thunder/Config.h @@ -25,6 +25,14 @@ namespace Thunder { namespace PluginHost { + + enum class PostMortemDataSink : uint8_t { + DISABLED = 0, + FILE = 1, + LOG = 2, + ALL = 3 + }; + /** * IMPORTANT: If updating this class to add/remove/modify configuration options, ensure * the documentation in docs/introduction/config.md is updated to reflect the changes! @@ -459,6 +467,8 @@ namespace PluginHost { Add(_T("volatilepath"), &VolatilePath); Add(_T("proxystubpath"), &ProxyStubPath); Add(_T("postmortempath"), &PostMortemPath); + Add(_T("postmortemworkerpoolsink"), &PostMortemWorkerPoolSink); + Add(_T("postmortemcallstacksink"), &PostMortemCallstackSink); Add(_T("communicator"), &Communicator); Add(_T("signature"), &Signature); Add(_T("idletime"), &IdleTime); @@ -515,6 +525,8 @@ namespace PluginHost { Core::JSON::String VolatilePath; Core::JSON::String ProxyStubPath; Core::JSON::String PostMortemPath; + Core::JSON::EnumType PostMortemWorkerPoolSink; + Core::JSON::EnumType PostMortemCallstackSink; Core::JSON::String Communicator; Core::JSON::String Redirect; Core::JSON::String Signature; @@ -695,6 +707,8 @@ namespace PluginHost { , _proxyStubPath() , _observableProxyStubPath() , _postMortemPath() + , _workerPoolSink(static_cast(THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT)) + , _callstackSink(static_cast(THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT)) , _pluginConfigPath() , _accessor() , _communicator() @@ -808,6 +822,12 @@ namespace PluginHost { _pluginConfigPath = Core::Directory::Normalize(config.Observe.PluginConfigPath.Value()); } _postMortemPath = Core::Directory::Normalize(config.PostMortemPath.Value()); + _workerPoolSink = config.PostMortemWorkerPoolSink.IsSet() + ? config.PostMortemWorkerPoolSink.Value() + : static_cast(THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT); + _callstackSink = config.PostMortemCallstackSink.IsSet() + ? config.PostMortemCallstackSink.Value() + : static_cast(THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT); _appPath = Core::Directory::Normalize(Core::File::PathName(Core::ProcessInfo().Executable())); _hashKey = config.Signature.Value(); _communicator = Core::NodeId(config.Communicator.Value().c_str()); @@ -1039,6 +1059,14 @@ namespace PluginHost { { return (_postMortemPath); } + inline PostMortemDataSink PostMortemWorkerPoolSink() const + { + return (_workerPoolSink); + } + inline PostMortemDataSink PostMortemCallstackSink() const + { + return (_callstackSink); + } inline bool PostMortemAllowed(PluginHost::IShell::reason why) const { std::list::const_iterator index(std::find(_reasons.begin(), _reasons.end(), why)); @@ -1334,6 +1362,8 @@ namespace PluginHost { string _proxyStubPath; string _observableProxyStubPath; string _postMortemPath; + PostMortemDataSink _workerPoolSink; + PostMortemDataSink _callstackSink; string _pluginConfigPath; Core::NodeId _accessor; Core::NodeId _communicator; diff --git a/Source/Thunder/GenericConfig.cmake b/Source/Thunder/GenericConfig.cmake index 16d00588bc..641a8502ca 100644 --- a/Source/Thunder/GenericConfig.cmake +++ b/Source/Thunder/GenericConfig.cmake @@ -89,6 +89,8 @@ map_set(${CONFIG} systempath ${SYSTEM_PATH}) map_set(${CONFIG} extensionpath ${EXTENSION_PATH}) map_set(${CONFIG} proxystubpath ${PROXYSTUB_PATH}) map_set(${CONFIG} postmortempath ${POSTMORTEM_PATH}) +map_set(${CONFIG} postmortemworkerpoolsink ${POSTMORTEM_WORKERPOOL_SINK}) +map_set(${CONFIG} postmortemcallstacksink ${POSTMORTEM_CALLSTACK_SINK}) map_set(${CONFIG} redirect "/Service/Controller/UI/index.html") map_set(${CONFIG} ethernetcard ${ETHERNETCARD_NAME}) if(NOT COMMUNICATOR STREQUAL "") diff --git a/Source/Thunder/PluginServer.cpp b/Source/Thunder/PluginServer.cpp index b87ca88d17..c483d51cb4 100644 --- a/Source/Thunder/PluginServer.cpp +++ b/Source/Thunder/PluginServer.cpp @@ -51,6 +51,15 @@ ENUM_CONVERSION_BEGIN(PluginHost::InputHandler::type) ENUM_CONVERSION_END(PluginHost::InputHandler::type) +ENUM_CONVERSION_BEGIN(PluginHost::PostMortemDataSink) + + { PluginHost::PostMortemDataSink::DISABLED, _TXT("Disabled") }, + { PluginHost::PostMortemDataSink::FILE, _TXT("File") }, + { PluginHost::PostMortemDataSink::LOG, _TXT("Log") }, + { PluginHost::PostMortemDataSink::ALL, _TXT("All") }, + +ENUM_CONVERSION_END(PluginHost::PostMortemDataSink) + namespace PluginHost { // // STATIC declarations diff --git a/Source/Thunder/PluginServer.h b/Source/Thunder/PluginServer.h index b1c02d79ca..709da94e95 100644 --- a/Source/Thunder/PluginServer.h +++ b/Source/Thunder/PluginServer.h @@ -5689,35 +5689,63 @@ namespace PluginHost { return (_connections.Connection(id)); } inline void DumpMetadata() { - PostMortemData data; - _dispatcher.Snapshot(data.WorkerPool); + const PluginHost::PostMortemDataSink wpSink = _config.PostMortemWorkerPoolSink(); + const PluginHost::PostMortemDataSink csSink = _config.PostMortemCallstackSink(); - Core::JSON::ArrayType::Iterator index(data.WorkerPool.ThreadPoolRuns.Elements()); + if ((wpSink != PluginHost::PostMortemDataSink::DISABLED) || (csSink != PluginHost::PostMortemDataSink::DISABLED)) { - while (index.Next() == true) { + PostMortemData data; + _dispatcher.Snapshot(data.WorkerPool); - std::list stackList; - if (index.Current().Id.Value() != 0) { - ::DumpCallStack(PluginHost::Metadata::ThreadId(index.Current().Id.Value()), stackList); - } + Core::JSON::ArrayType::Iterator index(data.WorkerPool.ThreadPoolRuns.Elements()); + + while (index.Next() == true) { + + const uint64_t threadIdRaw = index.Current().Id.Value(); + + std::list stackList; + if (threadIdRaw != 0) { + ::DumpCallStack(PluginHost::Metadata::ThreadId(threadIdRaw), stackList); + } + + if (csSink == PluginHost::PostMortemDataSink::LOG || csSink == PluginHost::PostMortemDataSink::ALL) { + for (const Core::callstack_info& entry : stackList) { + const char* sym = entry.function.empty() ? "Unknown symbol" : entry.function.c_str(); + SYSLOG(Logging::Shutdown, (_T("[%s]:[%s]:[%u]:[%p]"), + entry.module.c_str(), sym, static_cast(entry.line), entry.address)); + } + } + + PostMortemData::Callstack dump; + dump.Id = threadIdRaw; - PostMortemData::Callstack dump; - dump.Id = index.Current().Id.Value(); + for (const Core::callstack_info& info : stackList) { + dump.Data.Add() = CallstackData(info); + } - for (const Core::callstack_info& info : stackList) { - dump.Data.Add() = CallstackData(info); + data.Callstacks.Add(dump); } - data.Callstacks.Add(dump); - } + if (csSink == PluginHost::PostMortemDataSink::FILE || csSink == PluginHost::PostMortemDataSink::ALL) { + DumpReadableMetadata(data, _config.PostMortemPath()); + } - // Drop the workerpool info (what is currently running and what is pending) to a file.. - Core::File dumpFile(_config.PostMortemPath() + "ThunderInternals.json"); - if (dumpFile.Create(false) == true) { - data.IElement::ToFile(dumpFile); - } + // Drop the workerpool info (what is currently running and what is pending) to a file.. + if (wpSink == PluginHost::PostMortemDataSink::FILE || wpSink == PluginHost::PostMortemDataSink::ALL) { + Core::File dumpFile(_config.PostMortemPath() + "ThunderInternals.json"); + if (dumpFile.Create(false) == true) { + data.IElement::ToFile(dumpFile); + } + } - DumpReadableMetadata(data); + if (wpSink == PluginHost::PostMortemDataSink::LOG || wpSink == PluginHost::PostMortemDataSink::ALL) { + string jsonContent; + data.IElement::ToString(jsonContent); + SYSLOG(Logging::Shutdown, (_T("WorkerPool snapshot start\n"))); + SYSLOG(Logging::Shutdown, (_T("[%s]\n"), jsonContent.c_str())); + SYSLOG(Logging::Shutdown, (_T("WorkerPool snapshot end\n"))); + } + } } inline ServiceMap& Services() { @@ -5797,7 +5825,7 @@ namespace PluginHost { } private: - void DumpReadableMetadata(PostMortemData& data) const + void DumpReadableMetadata(PostMortemData& data, const string& outputPath) const { string readableDump; @@ -5874,7 +5902,7 @@ namespace PluginHost { appendReadableStack(index.Current()); } - Core::File readableDumpFile(_config.PostMortemPath() + "ThunderInternals.txt"); + Core::File readableDumpFile(outputPath + "ThunderInternals.txt"); if (readableDumpFile.Create(false) == true) { const uint32_t dumpSize = static_cast(readableDump.length() * sizeof(string::value_type)); const uint32_t written = readableDumpFile.Write(reinterpret_cast(readableDump.c_str()), dumpSize); diff --git a/Source/Thunder/Thunder.conf.in b/Source/Thunder/Thunder.conf.in index 8d5af86bfe..59c342815d 100644 --- a/Source/Thunder/Thunder.conf.in +++ b/Source/Thunder/Thunder.conf.in @@ -12,6 +12,8 @@ systempath = '@SYSTEM_PATH@' extensionpath = '@EXTENSION_PATH@' proxystubpath = '@PROXYSTUB_PATH@' postmortempath = '@POSTMORTEM_PATH@' +postmortemworkerpoolsink = '@POSTMORTEM_WORKERPOOL_SINK@' +postmortemcallstacksink = '@POSTMORTEM_CALLSTACK_SINK@' redirect = '/Service/Controller/UI/index.html' ethernetcard = '@ETHERNETCARD_NAME@' communicator = '@COMMUNICATOR@' diff --git a/Source/Thunder/bridge.vcxproj b/Source/Thunder/bridge.vcxproj index f9a01c0b4d..71770e9211 100755 --- a/Source/Thunder/bridge.vcxproj +++ b/Source/Thunder/bridge.vcxproj @@ -119,7 +119,7 @@ true true true - APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=1;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=0;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true pch.h $(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons @@ -143,7 +143,7 @@ Level3 Disabled true - APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=1;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=0;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true pch.h $(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons @@ -165,7 +165,7 @@ Level3 Disabled true - APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=1;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=0;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true pch.h $(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons @@ -189,7 +189,7 @@ true true true - APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=1;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=0;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true pch.h $(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons diff --git a/Source/Thunder/params.config b/Source/Thunder/params.config index 57dad574bc..4342a9ea8a 100644 --- a/Source/Thunder/params.config +++ b/Source/Thunder/params.config @@ -13,6 +13,8 @@ systempath extensionpath proxystubpath postmortempath +postmortemworkerpoolsink +postmortemcallstacksink redirect ethernetcard communicator diff --git a/Tests/test_support/CMakeLists.txt b/Tests/test_support/CMakeLists.txt index 29a8b9f4f7..8f6a87bf21 100644 --- a/Tests/test_support/CMakeLists.txt +++ b/Tests/test_support/CMakeLists.txt @@ -43,6 +43,8 @@ target_compile_definitions(${TARGET} THREADPOOL_COUNT=${THREADPOOL_COUNT} DEFAULT_SYSTEM_PATH="${SYSTEM_PATH}" DEFAULT_PROXYSTUB_PATH="${PROXYSTUB_PATH}" + THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=1 + THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=0 ) target_compile_options(${TARGET} PRIVATE -Wno-psabi) diff --git a/docs/introduction/config.md b/docs/introduction/config.md index 83e1d5e73c..a7522f5616 100644 --- a/docs/introduction/config.md +++ b/docs/introduction/config.md @@ -31,6 +31,8 @@ This section documents the available options for Thunder. This is different from | volatilepath | Directory to store volatile temporary data.

Each plugin will have an associated directory underneath this corresponding to the callsign of the plugin | string | /tmp | /tmp/ | | proxystubpath | Directory to search for the generated proxy stub libraries | string | - | /usr/lib/thunder/proxystubs | | postmortempath | Directory to store debugging info (worker pool information, debug data) in the event of a plugin or server crash.

If breakpad is found during build, will store breakpad mindumps here | string | /opt/minidumps | /opt/minidumps | +| postmortemworkerpoolsink | Controls where the WorkerPool JSON snapshot is written on crash.

Allowed values: `FILE` (write ThunderInternals.json to `postmortempath`), `LOG` (emit via SYSLOG), `ALL` (both), `DISABLED` (skip) | string | FILE | FILE | +| postmortemcallstacksink | Controls where per-thread callstack output is written on crash.

Allowed values: `FILE` (write ThunderInternals.txt to `postmortempath`), `LOG` (emit via SYSLOG), `ALL` (both), `DISABLED` (skip) | string | DISABLED | DISABLED | | communicator | Socket to listen for COM-RPC messages. Can be a filesystem path on Linux for a Unix domain socket, or a TCP socket.

For unix sockets, the file permissions can be specified by adding a `|` followed by the numeric permissions | string | /tmp/communicator\|0777 | 127.0.0.1:4000 | | redirect | Redirect incoming HTTP requests to the root Thunder URL to this address (please note it must contain the resource that is required e.g. index.html ) | string | http://127.0.0.1/Service/Controller/UI/index.html | http://127.0.0.1/Service/Controller/UI/index.html | | idletime | Amount of time (in seconds) to wait before closing and cleaning up idle client connections. If no activity occurs over a connection for this time Thunder will close it. | integer | 180 | 180 |