From 9c9f20865cf4335f7b4f9486e52d521aa1d372f3 Mon Sep 17 00:00:00 2001 From: yawzhang Date: Thu, 24 Sep 2026 17:51:23 +0800 Subject: [PATCH] add health watchdog --- conanfile.py | 5 +- include/sisl/watchdog/watchdog_registry.hpp | 105 ++++++++++++++++ src/CMakeLists.txt | 1 + src/watchdog/CMakeLists.txt | 13 ++ src/watchdog/tests/test_watchdog_registry.cpp | 116 ++++++++++++++++++ src/watchdog/watchdog_registry.cpp | 90 ++++++++++++++ 6 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 include/sisl/watchdog/watchdog_registry.hpp create mode 100644 src/watchdog/CMakeLists.txt create mode 100644 src/watchdog/tests/test_watchdog_registry.cpp create mode 100644 src/watchdog/watchdog_registry.cpp diff --git a/conanfile.py b/conanfile.py index febfe321..39ec84c9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class SISLConan(ConanFile): name = "sisl" - version = "13.2.13" + version = "13.2.14" homepage = "https://github.com/eBay/sisl" description = "Library for fast data structures, utilities" @@ -127,6 +127,8 @@ def layout(self): self.cpp.package.components["file_watcher"].libs = ["sisl_file_watcher"] self.cpp.build.components["version"].libdirs = ["src/version"] self.cpp.package.components["version"].libs = ["sisl_version"] + self.cpp.build.components["watchdog"].libdirs = ["src/watchdog"] + self.cpp.package.components["watchdog"].libs = ["sisl_watchdog"] self.cpp.package.components["sisl"].libs = [""] if self.options.metrics: @@ -246,6 +248,7 @@ def package_info(self): "file_watcher", "sobject", "version", + "watchdog", ]) if self.options.metrics: diff --git a/include/sisl/watchdog/watchdog_registry.hpp b/include/sisl/watchdog/watchdog_registry.hpp new file mode 100644 index 00000000..d582d654 --- /dev/null +++ b/include/sisl/watchdog/watchdog_registry.hpp @@ -0,0 +1,105 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sisl { + +enum class WatchdogSeverity { + critical, +}; + +struct WatchdogFailure { + std::string name; + std::string details; + WatchdogSeverity severity; +}; + +class WatchdogRegistry; + +// Abstract base for all watchdog types. Non-copyable, non-movable. +// Destructor automatically deregisters from the registry. +class Watchdog { +public: + Watchdog() = default; + virtual ~Watchdog(); + Watchdog(const Watchdog&) = delete; + Watchdog& operator=(const Watchdog&) = delete; + Watchdog(Watchdog&&) = delete; + Watchdog& operator=(Watchdog&&) = delete; + + virtual std::optional< WatchdogFailure > check(std::chrono::steady_clock::time_point now) const = 0; + +private: + friend class WatchdogRegistry; + WatchdogRegistry* registry_{nullptr}; + uint64_t id_{0}; +}; + +// Watches a single operation that must complete within a fixed deadline. +class DeadlineWatchdog : public Watchdog { +public: + DeadlineWatchdog(std::string name, std::chrono::milliseconds limit); + std::optional< WatchdogFailure > check(std::chrono::steady_clock::time_point now) const override; + +protected: + std::atomic< std::chrono::steady_clock::time_point > anchor{}; + +private: + std::string name_; + std::chrono::milliseconds limit_{0}; +}; + +// For a recurring loop that must keep making progress. Call kick() each iteration to extend the deadline. +class LeaseWatchdog : public DeadlineWatchdog { +public: + LeaseWatchdog(std::string name, std::chrono::milliseconds limit) + : DeadlineWatchdog(std::move(name), limit) {} + void kick(); +}; + +// For signaling that the system is already in a known-bad state. Fails immediately on construction. +class BarkWatchdog : public Watchdog { +public: + BarkWatchdog(std::string name, std::string details); + std::optional< WatchdogFailure > check(std::chrono::steady_clock::time_point now) const override; + +private: + std::string name_; + std::string details_; +}; + +class WatchdogRegistry { +public: + WatchdogRegistry() = default; + + static WatchdogRegistry& instance() { + static WatchdogRegistry s_instance; + return s_instance; + } + + std::unique_ptr< DeadlineWatchdog > add_deadline(std::string name, std::chrono::milliseconds limit); + std::unique_ptr< LeaseWatchdog > add_lease(std::string name, std::chrono::milliseconds limit); + std::unique_ptr< BarkWatchdog > add_bark(std::string name, std::string details = {}); + + std::vector< WatchdogFailure > failures() const; + +private: + friend class Watchdog; + + uint64_t add(Watchdog* dog); + void remove(uint64_t id); + + mutable std::mutex mu_; + uint64_t next_id_{1}; + std::unordered_map< uint64_t, Watchdog* > dogs_; // non-owning; caller owns via unique_ptr +}; + +} // namespace sisl diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6d937dd6..76f218d8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,6 +22,7 @@ add_subdirectory(options) add_subdirectory(sobject) add_subdirectory(file_watcher) add_subdirectory(version) +add_subdirectory(watchdog) find_package(folly QUIET) if (${folly_FOUND}) diff --git a/src/watchdog/CMakeLists.txt b/src/watchdog/CMakeLists.txt new file mode 100644 index 00000000..e17faa48 --- /dev/null +++ b/src/watchdog/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.11) + +add_library(sisl_watchdog) +target_sources(sisl_watchdog PRIVATE + watchdog_registry.cpp +) + +add_executable(test_watchdog_registry) +target_sources(test_watchdog_registry PRIVATE + tests/test_watchdog_registry.cpp +) +target_link_libraries(test_watchdog_registry sisl_watchdog GTest::gtest) +add_test(NAME WatchdogRegistryTest COMMAND test_watchdog_registry) diff --git a/src/watchdog/tests/test_watchdog_registry.cpp b/src/watchdog/tests/test_watchdog_registry.cpp new file mode 100644 index 00000000..e6502dfc --- /dev/null +++ b/src/watchdog/tests/test_watchdog_registry.cpp @@ -0,0 +1,116 @@ +#include + +#include + +#include +#include + +using namespace sisl; +using namespace std::chrono_literals; + +namespace { + +bool wait_until_failed(WatchdogRegistry const& registry, std::chrono::milliseconds budget) { + auto const deadline = std::chrono::steady_clock::now() + budget; + while (std::chrono::steady_clock::now() < deadline) { + if (!registry.failures().empty()) { return true; } + std::this_thread::sleep_for(5ms); + } + return !registry.failures().empty(); +} + +} // namespace + +TEST(WatchdogRegistry, EmptyRegistryHasNoFailures) { + WatchdogRegistry registry; + EXPECT_TRUE(registry.failures().empty()); +} + +TEST(WatchdogRegistry, DeadlineNotFailedBeforeLimit) { + WatchdogRegistry registry; + auto wd = registry.add_deadline("gc.iteration", 5s); + (void)wd; + EXPECT_TRUE(registry.failures().empty()); +} + +TEST(WatchdogRegistry, DeadlineFailsAfterLimitAndClearsOnDrop) { + WatchdogRegistry registry; + { + auto wd = registry.add_deadline("gc.iteration", 20ms); + (void)wd; + EXPECT_TRUE(wait_until_failed(registry, 500ms)); + auto const failures = registry.failures(); + ASSERT_EQ(failures.size(), 1u); + EXPECT_EQ(failures[0].name, "gc.iteration"); + EXPECT_NE(failures[0].details.find("last seen"), std::string::npos); + EXPECT_EQ(failures[0].severity, WatchdogSeverity::critical); + } + EXPECT_TRUE(registry.failures().empty()); +} + +TEST(WatchdogRegistry, LeaseKickExtendsDeadline) { + WatchdogRegistry registry; + auto lease = registry.add_lease("heartbeat.loop", 80ms); + std::this_thread::sleep_for(20ms); + lease->kick(); + EXPECT_TRUE(registry.failures().empty()); + EXPECT_TRUE(wait_until_failed(registry, 500ms)); + ASSERT_EQ(registry.failures().size(), 1u); + EXPECT_EQ(registry.failures()[0].name, "heartbeat.loop"); +} + +TEST(WatchdogRegistry, LeaseFailsAndClearsOnDrop) { + WatchdogRegistry registry; + { + auto lease = registry.add_lease("heartbeat.loop", 20ms); + (void)lease; + EXPECT_TRUE(wait_until_failed(registry, 500ms)); + EXPECT_EQ(registry.failures().size(), 1u); + } + EXPECT_TRUE(registry.failures().empty()); +} + +TEST(WatchdogRegistry, BarkFailsImmediatelyAndClearsOnDrop) { + WatchdogRegistry registry; + { + auto bark = registry.add_bark("raft.system_exit", "exit_code=1"); + (void)bark; + auto const failures = registry.failures(); + ASSERT_EQ(failures.size(), 1u); + EXPECT_EQ(failures[0].name, "raft.system_exit"); + EXPECT_EQ(failures[0].details, "exit_code=1"); + EXPECT_EQ(failures[0].severity, WatchdogSeverity::critical); + } + EXPECT_TRUE(registry.failures().empty()); +} + +TEST(WatchdogRegistry, MoveTransfersRegistration) { + WatchdogRegistry registry; + std::unique_ptr< BarkWatchdog > moved; + { + auto bark = registry.add_bark("raft.system_exit", "exit_code=2"); + moved = std::move(bark); + EXPECT_EQ(registry.failures().size(), 1u); + } + EXPECT_EQ(registry.failures().size(), 1u); + moved.reset(); + EXPECT_TRUE(registry.failures().empty()); +} + +TEST(WatchdogRegistry, MultipleCriticalFailures) { + WatchdogRegistry registry; + auto bark = registry.add_bark("raft.system_exit", "exit_code=1"); + auto wd = registry.add_deadline("stuck", 20ms); + (void)bark; + (void)wd; + auto const deadline = std::chrono::steady_clock::now() + 500ms; + while (std::chrono::steady_clock::now() < deadline && registry.failures().size() < 2u) { + std::this_thread::sleep_for(5ms); + } + EXPECT_EQ(registry.failures().size(), 2u); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/watchdog/watchdog_registry.cpp b/src/watchdog/watchdog_registry.cpp new file mode 100644 index 00000000..71f8b7e1 --- /dev/null +++ b/src/watchdog/watchdog_registry.cpp @@ -0,0 +1,90 @@ +#include + +namespace sisl { + +// Watchdog base + +Watchdog::~Watchdog() { + if (registry_ == nullptr) { return; } + registry_->remove(id_); + registry_ = nullptr; + id_ = 0; +} + +// DeadlineWatchdog + +DeadlineWatchdog::DeadlineWatchdog(std::string name, std::chrono::milliseconds limit) + : name_(std::move(name)), limit_(limit) { + anchor.store(std::chrono::steady_clock::now()); +} + +std::optional< WatchdogFailure > DeadlineWatchdog::check(std::chrono::steady_clock::time_point now) const { + auto const a = anchor.load(); + if (now <= a + limit_) { return std::nullopt; } + auto const elapsed_ms = std::chrono::duration_cast< std::chrono::milliseconds >(now - a).count(); + auto const limit_ms = std::chrono::duration_cast< std::chrono::milliseconds >(limit_).count(); + auto const details = + "last seen " + std::to_string(elapsed_ms) + "ms ago (limit=" + std::to_string(limit_ms) + "ms)"; + return WatchdogFailure{name_, details, WatchdogSeverity::critical}; +} + +// LeaseWatchdog + +void LeaseWatchdog::kick() { anchor.store(std::chrono::steady_clock::now()); } + +// BarkWatchdog + +BarkWatchdog::BarkWatchdog(std::string name, std::string details) + : name_(std::move(name)), details_(std::move(details)) {} + +std::optional< WatchdogFailure > BarkWatchdog::check(std::chrono::steady_clock::time_point) const { + return WatchdogFailure{name_, details_, WatchdogSeverity::critical}; +} + +// WatchdogRegistry + +uint64_t WatchdogRegistry::add(Watchdog* dog) { + std::lock_guard< std::mutex > lock(mu_); + auto const id = next_id_++; + dog->registry_ = this; + dog->id_ = id; + dogs_.emplace(id, dog); + return id; +} + +void WatchdogRegistry::remove(uint64_t id) { + std::lock_guard< std::mutex > lock(mu_); + dogs_.erase(id); +} + +std::unique_ptr< DeadlineWatchdog > WatchdogRegistry::add_deadline(std::string name, std::chrono::milliseconds limit) { + auto dog = std::make_unique< DeadlineWatchdog >(std::move(name), limit); + add(dog.get()); + return dog; +} + +std::unique_ptr< LeaseWatchdog > WatchdogRegistry::add_lease(std::string name, + std::chrono::milliseconds limit) { + auto dog = std::make_unique< LeaseWatchdog >(std::move(name), limit); + add(dog.get()); + return dog; +} + +std::unique_ptr< BarkWatchdog > WatchdogRegistry::add_bark(std::string name, std::string details) { + auto dog = std::make_unique< BarkWatchdog >(std::move(name), std::move(details)); + add(dog.get()); + return dog; +} + +std::vector< WatchdogFailure > WatchdogRegistry::failures() const { + std::vector< WatchdogFailure > out; + auto const now = std::chrono::steady_clock::now(); + std::lock_guard< std::mutex > lock(mu_); + out.reserve(dogs_.size()); + for (auto const& entry : dogs_) { + if (auto f = entry.second->check(now)) { out.push_back(std::move(*f)); } + } + return out; +} + +} // namespace sisl