Add configurable sinks to postmortem WorkerPool and callstack dumps

Source/Thunder/CMakeLists.txt

@@ -22,6 +22,27 @@ get_filename_component(TARGET ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 set(THREADPOOL_COUNT "4" CACHE STRING "The number of threads in the thread pool")
 set(ENABLE_TRACING_MODULES ""  CACHE STRING "A space separated list of specific tracing modules to be enabled at start.")
 
+macro(thunder_postmortem_sink_value VAR_IN VAR_OUT)
+    string(TOUPPER "${${VAR_IN}}" _s)
+    if (_s STREQUAL "LOG")
+        set(${VAR_OUT} 1)
+    elseif(_s STREQUAL "ALL")
+        set(${VAR_OUT} 2)
+    elseif(_s STREQUAL "DISABLED")
+        set(${VAR_OUT} 3)
+    else()
+        set(${VAR_OUT} 0)
+    endif()
+endmacro()
+
+set(POSTMORTEM_WORKERPOOL_SINK "FILE" CACHE STRING "Default WorkerPool JSON dump sink on crash (FILE | LOG | ALL | DISABLED)")
+set(POSTMORTEM_CALLSTACK_SINK "DISABLED" CACHE STRING "Default inline callstack print sink on crash (FILE | LOG | ALL | DISABLED)")
+set_property(CACHE POSTMORTEM_WORKERPOOL_SINK PROPERTY STRINGS FILE LOG ALL DISABLED)
+set_property(CACHE POSTMORTEM_CALLSTACK_SINK  PROPERTY STRINGS FILE LOG ALL DISABLED)
+
+thunder_postmortem_sink_value(POSTMORTEM_WORKERPOOL_SINK _wp_sink)
+thunder_postmortem_sink_value(POSTMORTEM_CALLSTACK_SINK  _cs_sink)
+
 option(SKIP_PERSISTENT_PATH_INSTALL "This will skip installing the persistent path and its privileges. Needed for local build/installs" OFF)
 
 add_executable(${TARGET}
@@ -38,6 +59,8 @@ target_compile_definitions(${TARGET}
           NAMESPACE=${NAMESPACE}
           APPLICATION_NAME=${TARGET}
           THREADPOOL_COUNT=${THREADPOOL_COUNT}
+          THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=${_wp_sink}
+          THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=${_cs_sink}
         )
 
 target_compile_options (${TARGET} PRIVATE -Wno-psabi)

Source/Thunder/Config.h

@@ -25,6 +25,25 @@
 namespace Thunder {
 
 namespace PluginHost {
+
+    enum class PostMortemDataSink : uint8_t {
+        FILE     = 0,
+        LOG      = 1,
+        ALL      = 2,
+        DISABLED = 3
+    };
+
+    static inline PostMortemDataSink ParsePostMortemDataSink(const string& input, PostMortemDataSink fallback)
+    {
+        string s(input);
+        std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+        if      (s == _T("log"))      return PostMortemDataSink::LOG;
+        else if (s == _T("all"))      return PostMortemDataSink::ALL;
+        else if (s == _T("disabled")) return PostMortemDataSink::DISABLED;
+        else if (s == _T("file"))     return PostMortemDataSink::FILE;
+        return fallback;
+    }
+
     /**
      * IMPORTANT: If updating this class to add/remove/modify configuration options, ensure
      * the documentation in docs/introduction/config.md is updated to reflect the changes!
@@ -459,6 +478,9 @@ namespace PluginHost {
                 Add(_T("volatilepath"), &VolatilePath);
                 Add(_T("proxystubpath"), &ProxyStubPath);
                 Add(_T("postmortempath"), &PostMortemPath);
+                Add(_T("postmortemworkerpoolsink"), &PostMortemWorkerPoolSink);
+                Add(_T("postmortemcallstacksink"), &PostMortemCallstackSink);
+                Add(_T("postmortemcallstackdumppath"), &PostMortemCallstackDumpPath);
                 Add(_T("communicator"), &Communicator);
                 Add(_T("signature"), &Signature);
                 Add(_T("idletime"), &IdleTime);
@@ -515,6 +537,9 @@ namespace PluginHost {
             Core::JSON::String VolatilePath;
             Core::JSON::String ProxyStubPath;
             Core::JSON::String PostMortemPath;
+            Core::JSON::String PostMortemWorkerPoolSink;
+            Core::JSON::String PostMortemCallstackSink;
+            Core::JSON::String PostMortemCallstackDumpPath;
             Core::JSON::String Communicator;
             Core::JSON::String Redirect;
             Core::JSON::String Signature;
@@ -695,6 +720,9 @@ namespace PluginHost {
             , _proxyStubPath()
             , _observableProxyStubPath()
             , _postMortemPath()
+            , _workerPoolSink(static_cast<PostMortemDataSink>(THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT))
+            , _callstackSink(static_cast<PostMortemDataSink>(THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT))
+            , _callstackDumpPath()
             , _pluginConfigPath()
             , _accessor()
             , _communicator()
@@ -808,6 +836,17 @@ namespace PluginHost {
                     _pluginConfigPath = Core::Directory::Normalize(config.Observe.PluginConfigPath.Value());
                 }
                 _postMortemPath = Core::Directory::Normalize(config.PostMortemPath.Value());
+                _callstackDumpPath = config.PostMortemCallstackDumpPath.IsSet()
+                    ? Core::Directory::Normalize(config.PostMortemCallstackDumpPath.Value())
+                    : _postMortemPath;
+                _workerPoolSink = config.PostMortemWorkerPoolSink.IsSet()
+                    ? ParsePostMortemDataSink(config.PostMortemWorkerPoolSink.Value(),
+                                              PostMortemDataSink::FILE)
+                    : static_cast<PostMortemDataSink>(THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT);
+                _callstackSink = config.PostMortemCallstackSink.IsSet()
+                    ? ParsePostMortemDataSink(config.PostMortemCallstackSink.Value(),
+                                              PostMortemDataSink::DISABLED)
+                    : static_cast<PostMortemDataSink>(THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT);
                 _appPath = Core::Directory::Normalize(Core::File::PathName(Core::ProcessInfo().Executable()));
                 _hashKey = config.Signature.Value();
                 _communicator = Core::NodeId(config.Communicator.Value().c_str());
@@ -1039,6 +1078,18 @@ namespace PluginHost {
         {
             return (_postMortemPath);
         }
+        inline PostMortemDataSink PostMortemWorkerPoolSink() const
+        {
+            return (_workerPoolSink);
+        }
+        inline PostMortemDataSink PostMortemCallstackSink() const
+        {
+            return (_callstackSink);
+        }
+        inline const string& PostMortemCallstackDumpPath() const
+        {
+            return (_callstackDumpPath);
+        }
         inline bool PostMortemAllowed(PluginHost::IShell::reason why) const
         {
             std::list<PluginHost::IShell::reason>::const_iterator index(std::find(_reasons.begin(), _reasons.end(), why));
@@ -1334,6 +1385,9 @@ namespace PluginHost {
         string _proxyStubPath;
         string _observableProxyStubPath;
         string _postMortemPath;
+        PostMortemDataSink _workerPoolSink;
+        PostMortemDataSink _callstackSink;
+        string _callstackDumpPath;
         string _pluginConfigPath;
         Core::NodeId _accessor;
         Core::NodeId _communicator;

Source/Thunder/GenericConfig.cmake

@@ -31,6 +31,7 @@ set(WEBSERVER_PATH "${CMAKE_INSTALL_FULL_LOCALSTATEDIR}/www/html" CACHE PATH "Ro
 set(WEBSERVER_PORT 8080 CACHE STRING "Port for the HTTP server")
 set(PROXYSTUB_PATH "${CMAKE_INSTALL_FULL_LIBDIR}/${NAMESPACE_LIB}/proxystubs" CACHE PATH "Proxy stub path")
 set(POSTMORTEM_PATH "/opt/minidumps" CACHE PATH "Core file path to do the postmortem of the crash")
+set(POSTMORTEM_CALLSTACK_DUMP_PATH "${POSTMORTEM_PATH}" CACHE PATH "Path for callstack dump file (defaults to POSTMORTEM_PATH)")
 set(MESSAGECONTROL_PATH "MessageDispatcher" CACHE STRING "MessageControl base path to create message files")
 set(MESSAGING_PORT 0 CACHE STRING "The port for the messaging")
 set(MESSAGING_STDOUT false CACHE STRING "Enable message rederict from stdout")
@@ -89,6 +90,9 @@ map_set(${CONFIG} systempath ${SYSTEM_PATH})
 map_set(${CONFIG} extensionpath ${EXTENSION_PATH})
 map_set(${CONFIG} proxystubpath ${PROXYSTUB_PATH})
 map_set(${CONFIG} postmortempath ${POSTMORTEM_PATH})
+map_set(${CONFIG} postmortemworkerpoolsink ${POSTMORTEM_WORKERPOOL_SINK})
+map_set(${CONFIG} postmortemcallstacksink ${POSTMORTEM_CALLSTACK_SINK})
+map_set(${CONFIG} postmortemcallstackdumppath ${POSTMORTEM_CALLSTACK_DUMP_PATH})
 map_set(${CONFIG} redirect "/Service/Controller/UI/index.html")
 map_set(${CONFIG} ethernetcard ${ETHERNETCARD_NAME})
 if(NOT COMMUNICATOR STREQUAL "")

Source/Thunder/PluginServer.h

@@ -5694,15 +5694,29 @@ namespace PluginHost {
 
             Core::JSON::ArrayType<Metadata::Server::Minion>::Iterator index(data.WorkerPool.ThreadPoolRuns.Elements());
 
+            const PluginHost::PostMortemDataSink wpSink = _config.PostMortemWorkerPoolSink();
+            const PluginHost::PostMortemDataSink csSink = _config.PostMortemCallstackSink();
+
             while (index.Next() == true) {
 
+                const uint64_t threadIdRaw = index.Current().Id.Value();
+
                 std::list<Core::callstack_info> stackList;
-                if (index.Current().Id.Value() != 0) {
-                    ::DumpCallStack(PluginHost::Metadata::ThreadId(index.Current().Id.Value()), stackList);
+                if (threadIdRaw != 0) {
+                    ::DumpCallStack(PluginHost::Metadata::ThreadId(threadIdRaw), stackList);
+                }
+
+                if (csSink == PluginHost::PostMortemDataSink::LOG || csSink == PluginHost::PostMortemDataSink::ALL) {
+                    for (const Core::callstack_info& entry : stackList) {
+                        const char* sym = entry.function.empty() ? "Unknown symbol" : entry.function.c_str();
+                        fprintf(stderr, "[%s]:[%s]:[%d]:[%p]\n",
+                                entry.module.c_str(), sym, entry.line, entry.address);
+                    }
+                    fflush(stderr);
                 }
 
                 PostMortemData::Callstack dump;
-                dump.Id = index.Current().Id.Value();
+                dump.Id = threadIdRaw;
 
                 for (const Core::callstack_info& info : stackList) {
                     dump.Data.Add() = CallstackData(info);
@@ -5711,13 +5725,26 @@ namespace PluginHost {
                 data.Callstacks.Add(dump);
             }
 
+            if (csSink == PluginHost::PostMortemDataSink::FILE || csSink == PluginHost::PostMortemDataSink::ALL) {
+                DumpReadableMetadata(data, _config.PostMortemCallstackDumpPath());
+            }
+
             // Drop the workerpool info (what is currently running and what is pending) to a file..
-            Core::File dumpFile(_config.PostMortemPath() + "ThunderInternals.json");
-            if (dumpFile.Create(false) == true) {
-                data.IElement::ToFile(dumpFile);
+            if (wpSink == PluginHost::PostMortemDataSink::FILE || wpSink == PluginHost::PostMortemDataSink::ALL) {
+                Core::File dumpFile(_config.PostMortemPath() + "ThunderInternals.json");
+                if (dumpFile.Create(false) == true) {
+                    data.IElement::ToFile(dumpFile);
+                }
             }
 
-            DumpReadableMetadata(data);
+            if (wpSink == PluginHost::PostMortemDataSink::LOG || wpSink == PluginHost::PostMortemDataSink::ALL) {
+                string jsonContent;
+                data.IElement::ToString(jsonContent);
+                SYSLOG(Logging::Shutdown, (_T("WorkerPool snapshot start\n")));
+                SYSLOG(Logging::Shutdown, (_T("[%s]\n"), jsonContent.c_str()));
+                SYSLOG(Logging::Shutdown, (_T("WorkerPool snapshot end\n")));
+                fflush(stderr);
+            }
         }
         inline ServiceMap& Services()
         {
@@ -5797,7 +5824,7 @@ namespace PluginHost {
         }
 
     private:
-        void DumpReadableMetadata(PostMortemData& data) const
+        void DumpReadableMetadata(PostMortemData& data, const string& outputPath) const
         {
             string readableDump;
 
@@ -5874,7 +5901,7 @@ namespace PluginHost {
                 appendReadableStack(index.Current());
             }
 
-            Core::File readableDumpFile(_config.PostMortemPath() + "ThunderInternals.txt");
+            Core::File readableDumpFile(outputPath + "ThunderInternals.txt");
             if (readableDumpFile.Create(false) == true) {
                 const uint32_t dumpSize = static_cast<uint32_t>(readableDump.length() * sizeof(string::value_type));
                 const uint32_t written = readableDumpFile.Write(reinterpret_cast<const uint8_t*>(readableDump.c_str()), dumpSize);

Source/Thunder/Thunder.conf.in

@@ -12,6 +12,9 @@ systempath = '@SYSTEM_PATH@'
 extensionpath = '@EXTENSION_PATH@'
 proxystubpath = '@PROXYSTUB_PATH@'
 postmortempath = '@POSTMORTEM_PATH@'
+postmortemworkerpoolsink = '@POSTMORTEM_WORKERPOOL_SINK@'
+postmortemcallstacksink = '@POSTMORTEM_CALLSTACK_SINK@'
+postmortemcallstackdumppath = '@POSTMORTEM_CALLSTACK_DUMP_PATH@'
 redirect = '/Service/Controller/UI/index.html'
 ethernetcard = '@ETHERNETCARD_NAME@'
 communicator = '@COMMUNICATOR@'

Source/Thunder/bridge.vcxproj

@@ -119,7 +119,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=0;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=3;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <AdditionalIncludeDirectories>$(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons</AdditionalIncludeDirectories>
@@ -143,7 +143,7 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=0;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=3;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <AdditionalIncludeDirectories>$(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons</AdditionalIncludeDirectories>
@@ -165,7 +165,7 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=0;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=3;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <AdditionalIncludeDirectories>$(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons</AdditionalIncludeDirectories>
@@ -189,7 +189,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>APPLICATION_NAME=Thunder;__CORE_MESSAGING__;_CRT_SECURE_NO_WARNINGS;_WINDOWS;THREADPOOL_COUNT=4;HOSTING_COMPROCESS=comprocess.exe;THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=0;THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=3;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <AdditionalIncludeDirectories>$(FrameworkPath)plugins;$(FrameworkPath);$(WindowsPath);$(WindowsPath)zlib;$(FrameworkPath)addons</AdditionalIncludeDirectories>

Source/Thunder/params.config

@@ -13,6 +13,9 @@ systempath
 extensionpath
 proxystubpath
 postmortempath
+postmortemworkerpoolsink
+postmortemcallstacksink
+postmortemcallstackdumppath
 redirect
 ethernetcard
 communicator

Tests/test_support/CMakeLists.txt

@@ -43,6 +43,8 @@ target_compile_definitions(${TARGET}
         THREADPOOL_COUNT=${THREADPOOL_COUNT}
         DEFAULT_SYSTEM_PATH="${SYSTEM_PATH}"
         DEFAULT_PROXYSTUB_PATH="${PROXYSTUB_PATH}"
+        THUNDER_POSTMORTEM_WORKERPOOL_SINK_DEFAULT=0
+        THUNDER_POSTMORTEM_CALLSTACK_SINK_DEFAULT=3
 )
 
 target_compile_options(${TARGET} PRIVATE -Wno-psabi)