add health watchdog

conanfile.py

@@ -9,7 +9,7 @@
 
 class SISLConan(ConanFile):
     name = "sisl"
-    version = "13.2.13"
+    version = "13.2.14"
 
     homepage = "https://github.com/eBay/sisl"
     description = "Library for fast data structures, utilities"
@@ -127,6 +127,8 @@ def layout(self):
         self.cpp.package.components["file_watcher"].libs = ["sisl_file_watcher"]
         self.cpp.build.components["version"].libdirs = ["src/version"]
         self.cpp.package.components["version"].libs = ["sisl_version"]
+        self.cpp.build.components["watchdog"].libdirs = ["src/watchdog"]
+        self.cpp.package.components["watchdog"].libs = ["sisl_watchdog"]
         self.cpp.package.components["sisl"].libs = [""]
 
         if self.options.metrics:
@@ -246,6 +248,7 @@ def package_info(self):
                 "file_watcher",
                 "sobject",
                 "version",
+                "watchdog",
                 ])
 
         if self.options.metrics:

include/sisl/watchdog/watchdog_registry.hpp

@@ -0,0 +1,105 @@
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace sisl {
+
+enum class WatchdogSeverity {
+    critical,
+};
+
+struct WatchdogFailure {
+    std::string name;
+    std::string details;
+    WatchdogSeverity severity;
+};
+
+class WatchdogRegistry;
+
+// Abstract base for all watchdog types. Non-copyable, non-movable.
+// Destructor automatically deregisters from the registry.
+class Watchdog {
+public:
+    Watchdog() = default;
+    virtual ~Watchdog();
+    Watchdog(const Watchdog&) = delete;
+    Watchdog& operator=(const Watchdog&) = delete;
+    Watchdog(Watchdog&&) = delete;
+    Watchdog& operator=(Watchdog&&) = delete;
+
+    virtual std::optional< WatchdogFailure > check(std::chrono::steady_clock::time_point now) const = 0;
+
+private:
+    friend class WatchdogRegistry;
+    WatchdogRegistry* registry_{nullptr};
+    uint64_t id_{0};
+};
+
+// Watches a single operation that must complete within a fixed deadline.
+class DeadlineWatchdog : public Watchdog {
+public:
+    DeadlineWatchdog(std::string name, std::chrono::milliseconds limit);
+    std::optional< WatchdogFailure > check(std::chrono::steady_clock::time_point now) const override;
+
+protected:
+    std::atomic< std::chrono::steady_clock::time_point > anchor{};
+
+private:
+    std::string name_;
+    std::chrono::milliseconds limit_{0};
+};
+
+// For a recurring loop that must keep making progress. Call kick() each iteration to extend the deadline.
+class LeaseWatchdog : public DeadlineWatchdog {
+public:
+    LeaseWatchdog(std::string name, std::chrono::milliseconds limit)
+        : DeadlineWatchdog(std::move(name), limit) {}
+    void kick();
+};
+
+// For signaling that the system is already in a known-bad state. Fails immediately on construction.
+class BarkWatchdog : public Watchdog {
+public:
+    BarkWatchdog(std::string name, std::string details);
+    std::optional< WatchdogFailure > check(std::chrono::steady_clock::time_point now) const override;
+
+private:
+    std::string name_;
+    std::string details_;
+};
+
+class WatchdogRegistry {
+public:
+    WatchdogRegistry() = default;
+
+    static WatchdogRegistry& instance() {
+        static WatchdogRegistry s_instance;
+        return s_instance;
+    }
+
+    std::unique_ptr< DeadlineWatchdog > add_deadline(std::string name, std::chrono::milliseconds limit);
+    std::unique_ptr< LeaseWatchdog > add_lease(std::string name, std::chrono::milliseconds limit);
+    std::unique_ptr< BarkWatchdog > add_bark(std::string name, std::string details = {});
+
+    std::vector< WatchdogFailure > failures() const;
+
+private:
+    friend class Watchdog;
+
+    uint64_t add(Watchdog* dog);
+    void remove(uint64_t id);
+
+    mutable std::mutex mu_;
+    uint64_t next_id_{1};
+    std::unordered_map< uint64_t, Watchdog* > dogs_; // non-owning; caller owns via unique_ptr
+};
+
+} // namespace sisl

src/CMakeLists.txt

@@ -22,6 +22,7 @@ add_subdirectory(options)
 add_subdirectory(sobject)
 add_subdirectory(file_watcher)
 add_subdirectory(version)
+add_subdirectory(watchdog)
 
 find_package(folly QUIET)
 if (${folly_FOUND})

src/watchdog/CMakeLists.txt

@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.11)
+
+add_library(sisl_watchdog)
+target_sources(sisl_watchdog PRIVATE
+    watchdog_registry.cpp
+)
+
+add_executable(test_watchdog_registry)
+target_sources(test_watchdog_registry PRIVATE
+    tests/test_watchdog_registry.cpp
+)
+target_link_libraries(test_watchdog_registry sisl_watchdog GTest::gtest)
+add_test(NAME WatchdogRegistryTest COMMAND test_watchdog_registry)

src/watchdog/tests/test_watchdog_registry.cpp

@@ -0,0 +1,116 @@
+#include <sisl/watchdog/watchdog_registry.hpp>
+
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <thread>
+
+using namespace sisl;
+using namespace std::chrono_literals;
+
+namespace {
+
+bool wait_until_failed(WatchdogRegistry const& registry, std::chrono::milliseconds budget) {
+    auto const deadline = std::chrono::steady_clock::now() + budget;
+    while (std::chrono::steady_clock::now() < deadline) {
+        if (!registry.failures().empty()) { return true; }
+        std::this_thread::sleep_for(5ms);
+    }
+    return !registry.failures().empty();
+}
+
+} // namespace
+
+TEST(WatchdogRegistry, EmptyRegistryHasNoFailures) {
+    WatchdogRegistry registry;
+    EXPECT_TRUE(registry.failures().empty());
+}
+
+TEST(WatchdogRegistry, DeadlineNotFailedBeforeLimit) {
+    WatchdogRegistry registry;
+    auto wd = registry.add_deadline("gc.iteration", 5s);
+    (void)wd;
+    EXPECT_TRUE(registry.failures().empty());
+}
+
+TEST(WatchdogRegistry, DeadlineFailsAfterLimitAndClearsOnDrop) {
+    WatchdogRegistry registry;
+    {
+        auto wd = registry.add_deadline("gc.iteration", 20ms);
+        (void)wd;
+        EXPECT_TRUE(wait_until_failed(registry, 500ms));
+        auto const failures = registry.failures();
+        ASSERT_EQ(failures.size(), 1u);
+        EXPECT_EQ(failures[0].name, "gc.iteration");
+        EXPECT_NE(failures[0].details.find("last seen"), std::string::npos);
+        EXPECT_EQ(failures[0].severity, WatchdogSeverity::critical);
+    }
+    EXPECT_TRUE(registry.failures().empty());
+}
+
+TEST(WatchdogRegistry, LeaseKickExtendsDeadline) {
+    WatchdogRegistry registry;
+    auto lease = registry.add_lease("heartbeat.loop", 80ms);
+    std::this_thread::sleep_for(20ms);
+    lease->kick();
+    EXPECT_TRUE(registry.failures().empty());
+    EXPECT_TRUE(wait_until_failed(registry, 500ms));
+    ASSERT_EQ(registry.failures().size(), 1u);
+    EXPECT_EQ(registry.failures()[0].name, "heartbeat.loop");
+}
+
+TEST(WatchdogRegistry, LeaseFailsAndClearsOnDrop) {
+    WatchdogRegistry registry;
+    {
+        auto lease = registry.add_lease("heartbeat.loop", 20ms);
+        (void)lease;
+        EXPECT_TRUE(wait_until_failed(registry, 500ms));
+        EXPECT_EQ(registry.failures().size(), 1u);
+    }
+    EXPECT_TRUE(registry.failures().empty());
+}
+
+TEST(WatchdogRegistry, BarkFailsImmediatelyAndClearsOnDrop) {
+    WatchdogRegistry registry;
+    {
+        auto bark = registry.add_bark("raft.system_exit", "exit_code=1");
+        (void)bark;
+        auto const failures = registry.failures();
+        ASSERT_EQ(failures.size(), 1u);
+        EXPECT_EQ(failures[0].name, "raft.system_exit");
+        EXPECT_EQ(failures[0].details, "exit_code=1");
+        EXPECT_EQ(failures[0].severity, WatchdogSeverity::critical);
+    }
+    EXPECT_TRUE(registry.failures().empty());
+}
+
+TEST(WatchdogRegistry, MoveTransfersRegistration) {
+    WatchdogRegistry registry;
+    std::unique_ptr< BarkWatchdog > moved;
+    {
+        auto bark = registry.add_bark("raft.system_exit", "exit_code=2");
+        moved = std::move(bark);
+        EXPECT_EQ(registry.failures().size(), 1u);
+    }
+    EXPECT_EQ(registry.failures().size(), 1u);
+    moved.reset();
+    EXPECT_TRUE(registry.failures().empty());
+}
+
+TEST(WatchdogRegistry, MultipleCriticalFailures) {
+    WatchdogRegistry registry;
+    auto bark = registry.add_bark("raft.system_exit", "exit_code=1");
+    auto wd = registry.add_deadline("stuck", 20ms);
+    (void)bark;
+    (void)wd;
+    auto const deadline = std::chrono::steady_clock::now() + 500ms;
+    while (std::chrono::steady_clock::now() < deadline && registry.failures().size() < 2u) {
+        std::this_thread::sleep_for(5ms);
+    }
+    EXPECT_EQ(registry.failures().size(), 2u);
+}
+
+int main(int argc, char** argv) {
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}

src/watchdog/watchdog_registry.cpp

@@ -0,0 +1,90 @@
+#include <sisl/watchdog/watchdog_registry.hpp>
+
+namespace sisl {
+
+// Watchdog base
+
+Watchdog::~Watchdog() {
+    if (registry_ == nullptr) { return; }
+    registry_->remove(id_);
+    registry_ = nullptr;
+    id_ = 0;
+}
+
+// DeadlineWatchdog
+
+DeadlineWatchdog::DeadlineWatchdog(std::string name, std::chrono::milliseconds limit)
+    : name_(std::move(name)), limit_(limit) {
+    anchor.store(std::chrono::steady_clock::now());
+}
+
+std::optional< WatchdogFailure > DeadlineWatchdog::check(std::chrono::steady_clock::time_point now) const {
+    auto const a = anchor.load();
+    if (now <= a + limit_) { return std::nullopt; }
+    auto const elapsed_ms = std::chrono::duration_cast< std::chrono::milliseconds >(now - a).count();
+    auto const limit_ms = std::chrono::duration_cast< std::chrono::milliseconds >(limit_).count();
+    auto const details =
+        "last seen " + std::to_string(elapsed_ms) + "ms ago (limit=" + std::to_string(limit_ms) + "ms)";
+    return WatchdogFailure{name_, details, WatchdogSeverity::critical};
+}
+
+// LeaseWatchdog
+
+void LeaseWatchdog::kick() { anchor.store(std::chrono::steady_clock::now()); }
+
+// BarkWatchdog
+
+BarkWatchdog::BarkWatchdog(std::string name, std::string details)
+    : name_(std::move(name)), details_(std::move(details)) {}
+
+std::optional< WatchdogFailure > BarkWatchdog::check(std::chrono::steady_clock::time_point) const {
+    return WatchdogFailure{name_, details_, WatchdogSeverity::critical};
+}
+
+// WatchdogRegistry
+
+uint64_t WatchdogRegistry::add(Watchdog* dog) {
+    std::lock_guard< std::mutex > lock(mu_);
+    auto const id = next_id_++;
+    dog->registry_ = this;
+    dog->id_ = id;
+    dogs_.emplace(id, dog);
+    return id;
+}
+
+void WatchdogRegistry::remove(uint64_t id) {
+    std::lock_guard< std::mutex > lock(mu_);
+    dogs_.erase(id);
+}
+
+std::unique_ptr< DeadlineWatchdog > WatchdogRegistry::add_deadline(std::string name, std::chrono::milliseconds limit) {
+    auto dog = std::make_unique< DeadlineWatchdog >(std::move(name), limit);
+    add(dog.get());
+    return dog;
+}
+
+std::unique_ptr< LeaseWatchdog > WatchdogRegistry::add_lease(std::string name,
+                                                             std::chrono::milliseconds limit) {
+    auto dog = std::make_unique< LeaseWatchdog >(std::move(name), limit);
+    add(dog.get());
+    return dog;
+}
+
+std::unique_ptr< BarkWatchdog > WatchdogRegistry::add_bark(std::string name, std::string details) {
+    auto dog = std::make_unique< BarkWatchdog >(std::move(name), std::move(details));
+    add(dog.get());
+    return dog;
+}
+
+std::vector< WatchdogFailure > WatchdogRegistry::failures() const {
+    std::vector< WatchdogFailure > out;
+    auto const now = std::chrono::steady_clock::now();
+    std::lock_guard< std::mutex > lock(mu_);
+    out.reserve(dogs_.size());
+    for (auto const& entry : dogs_) {
+        if (auto f = entry.second->check(now)) { out.push_back(std::move(*f)); }
+    }
+    return out;
+}
+
+} // namespace sisl