From 6581534971cc909ad465bb6ff65935a543c9e96c Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:06:29 +0530 Subject: [PATCH 01/21] metrics-watcher: scaffold package --- services/metrics-watcher/.dockerignore | 3 + services/metrics-watcher/package-lock.json | 1753 ++++++++++++++++++++ services/metrics-watcher/package.json | 20 + services/metrics-watcher/tsconfig.json | 15 + 4 files changed, 1791 insertions(+) create mode 100644 services/metrics-watcher/.dockerignore create mode 100644 services/metrics-watcher/package-lock.json create mode 100644 services/metrics-watcher/package.json create mode 100644 services/metrics-watcher/tsconfig.json diff --git a/services/metrics-watcher/.dockerignore b/services/metrics-watcher/.dockerignore new file mode 100644 index 0000000..d4d7e3f --- /dev/null +++ b/services/metrics-watcher/.dockerignore @@ -0,0 +1,3 @@ +node_modules +dist +*.log diff --git a/services/metrics-watcher/package-lock.json b/services/metrics-watcher/package-lock.json new file mode 100644 index 0000000..e8b6920 --- /dev/null +++ b/services/metrics-watcher/package-lock.json @@ -0,0 +1,1753 @@ +{ + "name": "metrics-watcher", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "metrics-watcher", + "version": "1.0.0", + "dependencies": { + "@modelcontextprotocol/sdk": "^1.30.0", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/node": "^22.5.0", + "tsx": "^4.19.0", + "typescript": "^5.5.4" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.6.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.2.tgz", + "integrity": "sha512-YH4ru+eOJxQABscKFfRCy9R7x9QFGdezclVMwwgFFndzS2Xnm0uo6B0ABZsLhcpeptGv2qvuJVWlQr9gQZoC3A==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.13.5", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.5.tgz", + "integrity": "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", + "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.10", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.10.tgz", + "integrity": "sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tsx": { + "version": "4.23.12", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", + "integrity": "sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/services/metrics-watcher/package.json b/services/metrics-watcher/package.json new file mode 100644 index 0000000..31b9ec2 --- /dev/null +++ b/services/metrics-watcher/package.json @@ -0,0 +1,20 @@ +{ + "name": "metrics-watcher", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc -p tsconfig.json", + "start": "node dist/server.js", + "test": "node --import tsx --test test/**/*.test.ts" + }, + "dependencies": { + "@modelcontextprotocol/sdk": "^1.30.0", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/node": "^22.5.0", + "tsx": "^4.19.0", + "typescript": "^5.5.4" + } +} diff --git a/services/metrics-watcher/tsconfig.json b/services/metrics-watcher/tsconfig.json new file mode 100644 index 0000000..c35bd02 --- /dev/null +++ b/services/metrics-watcher/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true + }, + "include": ["src"] +} From 6461e49f2e159df3a932c7f17f8d530b74ab72c9 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:09:20 +0530 Subject: [PATCH 02/21] metrics-watcher: add Prometheus query client --- services/metrics-watcher/src/prometheus.ts | 64 +++++++++++++++++++ .../metrics-watcher/test/prometheus.test.ts | 64 +++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 services/metrics-watcher/src/prometheus.ts create mode 100644 services/metrics-watcher/test/prometheus.test.ts diff --git a/services/metrics-watcher/src/prometheus.ts b/services/metrics-watcher/src/prometheus.ts new file mode 100644 index 0000000..818fd62 --- /dev/null +++ b/services/metrics-watcher/src/prometheus.ts @@ -0,0 +1,64 @@ +export interface RouteMetrics { + errorRatePercent: number | null; + avgLatencyMs: number | null; + requestCount: number; +} + +interface PrometheusQueryResult { + status: string; + data?: { + resultType: string; + result: Array<{ metric: Record; value: [number, string] }>; + }; +} + +async function query(baseUrl: string, promql: string, fetchImpl: typeof fetch): Promise { + const url = new URL("/api/v1/query", baseUrl); + url.searchParams.set("query", promql); + const res = await fetchImpl(url.toString()); + if (!res.ok) { + throw new Error(`Prometheus query failed: ${res.status} ${res.statusText}`); + } + const body = (await res.json()) as PrometheusQueryResult; + if (body.status !== "success") { + throw new Error(`Prometheus query returned status ${body.status}`); + } + const result = body.data?.result ?? []; + if (result.length === 0) return null; + return Number(result[0].value[1]); +} + +export async function queryRouteMetrics( + baseUrl: string, + route: string, + windowSeconds: number, + fetchImpl: typeof fetch = fetch, +): Promise { + const requestCountRaw = await query( + baseUrl, + `sum(increase(http_requests_total{route="${route}"}[${windowSeconds}s]))`, + fetchImpl, + ); + const requestCount = requestCountRaw ?? 0; + if (requestCount <= 0) { + return { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }; + } + + const errorRatePercent = await query( + baseUrl, + `100 * (sum(rate(http_requests_total{route="${route}",status=~"5.."}[${windowSeconds}s])) or vector(0)) / sum(rate(http_requests_total{route="${route}"}[${windowSeconds}s]))`, + fetchImpl, + ); + + const avgLatencySeconds = await query( + baseUrl, + `rate(http_request_duration_seconds_sum{route="${route}"}[${windowSeconds}s]) / rate(http_request_duration_seconds_count{route="${route}"}[${windowSeconds}s])`, + fetchImpl, + ); + + return { + errorRatePercent: errorRatePercent ?? 0, + avgLatencyMs: avgLatencySeconds !== null ? avgLatencySeconds * 1000 : null, + requestCount, + }; +} diff --git a/services/metrics-watcher/test/prometheus.test.ts b/services/metrics-watcher/test/prometheus.test.ts new file mode 100644 index 0000000..57debff --- /dev/null +++ b/services/metrics-watcher/test/prometheus.test.ts @@ -0,0 +1,64 @@ +// services/metrics-watcher/test/prometheus.test.ts +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { queryRouteMetrics } from "../src/prometheus.js"; + +function fakePrometheus(responses: Record) { + return async (url: string | URL) => { + const u = new URL(url); + const query = u.searchParams.get("query") ?? ""; + for (const [substring, value] of Object.entries(responses)) { + if (query.includes(substring)) { + return new Response( + JSON.stringify({ + status: "success", + data: { resultType: "vector", result: [{ metric: {}, value: [1700000000, value] }] }, + }), + { status: 200 }, + ); + } + } + // No matching substring: empty result vector (Prometheus's real shape for "no data"). + return new Response( + JSON.stringify({ status: "success", data: { resultType: "vector", result: [] } }), + { status: 200 }, + ); + }; +} + +test("queryRouteMetrics returns real numbers when Prometheus has data", async () => { + const fetchImpl = fakePrometheus({ + "increase(http_requests_total": "12", + "status=~\"5..\"": "25", + "http_request_duration_seconds_sum": "0.5", + }); + const result = await queryRouteMetrics("http://prometheus:9090", "/products", 60, fetchImpl as unknown as typeof fetch); + assert.equal(result.requestCount, 12); + assert.equal(result.errorRatePercent, 25); + assert.equal(result.avgLatencyMs, 500); +}); + +test("queryRouteMetrics returns nulls (not 0 or NaN) when there is no traffic in the window", async () => { + const fetchImpl = fakePrometheus({}); + const result = await queryRouteMetrics("http://prometheus:9090", "/health", 60, fetchImpl as unknown as typeof fetch); + assert.equal(result.requestCount, 0); + assert.equal(result.errorRatePercent, null); + assert.equal(result.avgLatencyMs, null); +}); + +test("queryRouteMetrics throws when Prometheus returns a non-2xx response", async () => { + const fetchImpl = async () => new Response("internal error", { status: 500, statusText: "Internal Server Error" }); + await assert.rejects( + () => queryRouteMetrics("http://prometheus:9090", "/products", 60, fetchImpl as unknown as typeof fetch), + /Prometheus query failed/, + ); +}); + +test("queryRouteMetrics throws when Prometheus responds with a non-success status body", async () => { + const fetchImpl = async () => + new Response(JSON.stringify({ status: "error", error: "bad query" }), { status: 200 }); + await assert.rejects( + () => queryRouteMetrics("http://prometheus:9090", "/products", 60, fetchImpl as unknown as typeof fetch), + /Prometheus query returned status/, + ); +}); From 7b01917e146f67c5d6f16ace05f375899b7fbc61 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:12:48 +0530 Subject: [PATCH 03/21] metrics-watcher: add severity classification and verdict logic --- services/metrics-watcher/src/severity.ts | 29 ++++++++++++ .../metrics-watcher/test/severity.test.ts | 47 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 services/metrics-watcher/src/severity.ts create mode 100644 services/metrics-watcher/test/severity.test.ts diff --git a/services/metrics-watcher/src/severity.ts b/services/metrics-watcher/src/severity.ts new file mode 100644 index 0000000..40b7da8 --- /dev/null +++ b/services/metrics-watcher/src/severity.ts @@ -0,0 +1,29 @@ +import type { RouteMetrics } from "./prometheus.js"; + +export const OBSERVED_HARD_ERROR_RATE_PERCENT = 40; +export const OBSERVED_NONE_ERROR_RATE_PERCENT = 1; + +export type ObservedSeverity = "hard" | "degraded" | "none"; +export type PredictedSeverity = "hard" | "degraded"; +export type Verdict = "matched" | "milder_than_predicted" | "worse_than_predicted"; + +export function classifyRoute(metrics: RouteMetrics): ObservedSeverity { + if (metrics.errorRatePercent === null) return "none"; + if (metrics.errorRatePercent >= OBSERVED_HARD_ERROR_RATE_PERCENT) return "hard"; + if (metrics.errorRatePercent <= OBSERVED_NONE_ERROR_RATE_PERCENT) return "none"; + return "degraded"; +} + +const SEVERITY_RANK: Record = { none: 0, degraded: 1, hard: 2 }; + +export function worstOf(severities: ObservedSeverity[]): ObservedSeverity { + return severities.reduce( + (worst, s) => (SEVERITY_RANK[s] > SEVERITY_RANK[worst] ? s : worst), + "none", + ); +} + +export function computeVerdict(predicted: PredictedSeverity, observed: ObservedSeverity): Verdict { + if (SEVERITY_RANK[observed] === SEVERITY_RANK[predicted]) return "matched"; + return SEVERITY_RANK[observed] > SEVERITY_RANK[predicted] ? "worse_than_predicted" : "milder_than_predicted"; +} diff --git a/services/metrics-watcher/test/severity.test.ts b/services/metrics-watcher/test/severity.test.ts new file mode 100644 index 0000000..f6d44cb --- /dev/null +++ b/services/metrics-watcher/test/severity.test.ts @@ -0,0 +1,47 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { classifyRoute, worstOf, computeVerdict } from "../src/severity.js"; + +test("classifyRoute: no traffic in window classifies as none", () => { + assert.equal(classifyRoute({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }), "none"); +}); + +test("classifyRoute: error rate at or above 40% classifies as hard", () => { + assert.equal(classifyRoute({ errorRatePercent: 40, avgLatencyMs: 10, requestCount: 5 }), "hard"); + assert.equal(classifyRoute({ errorRatePercent: 87, avgLatencyMs: 10, requestCount: 5 }), "hard"); +}); + +test("classifyRoute: error rate at or below 1% classifies as none", () => { + assert.equal(classifyRoute({ errorRatePercent: 0, avgLatencyMs: 10, requestCount: 5 }), "none"); + assert.equal(classifyRoute({ errorRatePercent: 1, avgLatencyMs: 10, requestCount: 5 }), "none"); +}); + +test("classifyRoute: error rate strictly between 1% and 40% classifies as degraded", () => { + assert.equal(classifyRoute({ errorRatePercent: 1.5, avgLatencyMs: 10, requestCount: 5 }), "degraded"); + assert.equal(classifyRoute({ errorRatePercent: 39.9, avgLatencyMs: 10, requestCount: 5 }), "degraded"); +}); + +test("worstOf: returns the highest-severity value present", () => { + assert.equal(worstOf(["none", "degraded", "none"]), "degraded"); + assert.equal(worstOf(["none", "degraded", "hard"]), "hard"); + assert.equal(worstOf(["none"]), "none"); +}); + +test("worstOf: empty array returns none", () => { + assert.equal(worstOf([]), "none"); +}); + +test("computeVerdict: observed matches predicted", () => { + assert.equal(computeVerdict("hard", "hard"), "matched"); + assert.equal(computeVerdict("degraded", "degraded"), "matched"); +}); + +test("computeVerdict: observed is milder than predicted", () => { + assert.equal(computeVerdict("hard", "degraded"), "milder_than_predicted"); + assert.equal(computeVerdict("hard", "none"), "milder_than_predicted"); + assert.equal(computeVerdict("degraded", "none"), "milder_than_predicted"); +}); + +test("computeVerdict: observed is worse than predicted", () => { + assert.equal(computeVerdict("degraded", "hard"), "worse_than_predicted"); +}); From 7db48a51670f14c6a03d3819a95c0b2aa7534e10 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:15:23 +0530 Subject: [PATCH 04/21] metrics-watcher: add observe_impact MCP tool --- services/metrics-watcher/src/tools.ts | 73 +++++++++++++++++++++ services/metrics-watcher/test/tools.test.ts | 70 ++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 services/metrics-watcher/src/tools.ts create mode 100644 services/metrics-watcher/test/tools.test.ts diff --git a/services/metrics-watcher/src/tools.ts b/services/metrics-watcher/src/tools.ts new file mode 100644 index 0000000..d45e137 --- /dev/null +++ b/services/metrics-watcher/src/tools.ts @@ -0,0 +1,73 @@ +// services/metrics-watcher/src/tools.ts +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { queryRouteMetrics, type RouteMetrics } from "./prometheus.js"; +import { classifyRoute, worstOf, computeVerdict, type ObservedSeverity, type Verdict } from "./severity.js"; + +export const KNOWN_ROUTES = ["/products", "/orders", "/health", "/metrics"] as const; +export type KnownRoute = (typeof KNOWN_ROUTES)[number]; + +const PROMETHEUS_URL = process.env.PROMETHEUS_URL ?? "http://prometheus:9090"; + +export interface ObserveImpactArgs { + predicted_severity: "hard" | "degraded"; + affected_routes: KnownRoute[]; + window_seconds: number; +} + +export interface ObserveImpactResult { + windowSeconds: number; + predictedSeverity: "hard" | "degraded"; + observedSeverity: ObservedSeverity; + verdict: Verdict; + routes: Record; +} + +type RouteMetricsFetcher = (route: KnownRoute) => Promise; + +// window_seconds varies per call, so the default fetcher is built per-call +// inside handleObserveImpact's default parameter (below), not as a single +// top-level constant. +export async function handleObserveImpact( + args: ObserveImpactArgs, + fetchRouteMetrics: RouteMetricsFetcher = (route) => queryRouteMetrics(PROMETHEUS_URL, route, args.window_seconds), +): Promise { + const routes = {} as Record; + for (const route of KNOWN_ROUTES) { + routes[route] = await fetchRouteMetrics(route); + } + + const affectedSeverities = args.affected_routes.map((route) => classifyRoute(routes[route])); + const observedSeverity = worstOf(affectedSeverities); + const verdict = computeVerdict(args.predicted_severity, observedSeverity); + + return { + windowSeconds: args.window_seconds, + predictedSeverity: args.predicted_severity, + observedSeverity, + verdict, + routes, + }; +} + +export function registerTools( + server: McpServer, + fetchRouteMetrics?: RouteMetricsFetcher, +): void { + server.registerTool( + "observe_impact", + { + title: "Observe real chaos-experiment impact", + description: + "Query Prometheus for checkout-api's real, currently-measured error rate and latency per route, and compare against a predicted severity from predict_blast_radius. Call this after a fault reverts, passing the same predicted_severity and affected routes you got from predict_blast_radius (stripped of any leading HTTP verb, e.g. 'GET /products' becomes '/products'), and the fault's duration_seconds as window_seconds.", + inputSchema: { + predicted_severity: z.enum(["hard", "degraded"]), + affected_routes: z.array(z.enum(KNOWN_ROUTES)).min(1), + window_seconds: z.number().int().positive(), + }, + }, + async (args) => ({ + content: [{ type: "text", text: JSON.stringify(await handleObserveImpact(args, fetchRouteMetrics)) }], + }), + ); +} diff --git a/services/metrics-watcher/test/tools.test.ts b/services/metrics-watcher/test/tools.test.ts new file mode 100644 index 0000000..049a56f --- /dev/null +++ b/services/metrics-watcher/test/tools.test.ts @@ -0,0 +1,70 @@ +// services/metrics-watcher/test/tools.test.ts +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { handleObserveImpact, registerTools } from "../src/tools.js"; + +test("handleObserveImpact reports hard severity and a matched verdict when the affected route is failing hard", async () => { + const result = await handleObserveImpact( + { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 60 }, + async () => ({ + "/products": { errorRatePercent: 87, avgLatencyMs: 5, requestCount: 20 }, + "/orders": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, + "/health": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, + "/metrics": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, + })["/products"], + ); + assert.equal(result.observedSeverity, "hard"); + assert.equal(result.verdict, "matched"); + assert.equal(result.routes["/products"].errorRatePercent, 87); +}); + +test("registerTools registers observe_impact on a real McpServer and validates input via zod", async () => { + const server = new McpServer({ name: "test-server", version: "1.0.0" }); + registerTools(server, async () => ({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 })); + + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "test-client", version: "1.0.0" }); + await Promise.all([client.connect(clientTransport), server.connect(serverTransport)]); + + const tools = await client.listTools(); + assert.deepEqual( + tools.tools.map((t) => t.name), + ["observe_impact"], + ); + + const good = await client.callTool({ + name: "observe_impact", + arguments: { predicted_severity: "degraded", affected_routes: ["/products"], window_seconds: 60 }, + }); + assert.equal(good.isError, undefined); + + const badSeverity = await client.callTool({ + name: "observe_impact", + arguments: { predicted_severity: "catastrophic", affected_routes: ["/products"], window_seconds: 60 }, + }); + assert.equal(badSeverity.isError, true); + + const badRoute = await client.callTool({ + name: "observe_impact", + arguments: { predicted_severity: "hard", affected_routes: ["/not-a-real-route"], window_seconds: 60 }, + }); + assert.equal(badRoute.isError, true); + + const emptyRoutes = await client.callTool({ + name: "observe_impact", + arguments: { predicted_severity: "hard", affected_routes: [], window_seconds: 60 }, + }); + assert.equal(emptyRoutes.isError, true); + + const badWindow = await client.callTool({ + name: "observe_impact", + arguments: { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 0 }, + }); + assert.equal(badWindow.isError, true); + + await client.close(); + await server.close(); +}); From 30af1e55503e26b3a8ff985945e57c7806e3c577 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:17:54 +0530 Subject: [PATCH 05/21] metrics-watcher: add server entrypoint --- services/metrics-watcher/src/server.ts | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 services/metrics-watcher/src/server.ts diff --git a/services/metrics-watcher/src/server.ts b/services/metrics-watcher/src/server.ts new file mode 100644 index 0000000..76ae17a --- /dev/null +++ b/services/metrics-watcher/src/server.ts @@ -0,0 +1,55 @@ +import http from "node:http"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { registerTools } from "./tools.js"; + +const PORT = Number(process.env.PORT ?? 3300); + +async function main() { + const mcpServer = new McpServer({ name: "chaos-notary-metrics-watcher", version: "1.0.0" }); + registerTools(mcpServer); + + // Stateless mode: each request is an independent JSON-RPC call. See + // services/mcp-server/src/server.ts (M2) for why — a stateful transport with a generated + // session ID rejects every request after the first client session closes, + // and every caller here (the acceptance script's cliCall.ts) is a + // short-lived, one-call-then-disconnect client. + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + }); + await mcpServer.connect(transport); + + const httpServer = http.createServer((req, res) => { + if (req.url === "/health") { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ status: "ok" })); + return; + } + if (req.url === "/mcp") { + // .catch(), not `void` alone: an unhandled rejection here would crash + // the whole process on a single malformed request (this exact bug + // was found and fixed in M2's server.ts during its Qodo review). + transport.handleRequest(req, res).catch((err) => { + console.error("error handling /mcp request", err); + if (!res.headersSent) { + res.writeHead(500, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "internal server error" })); + } else { + res.end(); + } + }); + return; + } + res.writeHead(404); + res.end(); + }); + + httpServer.listen(PORT, () => { + console.log(`chaos-notary metrics-watcher listening on :${PORT}`); + }); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From e63ef2bb97946ffd60085d6c3f6fc8dd830f559b Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:19:39 +0530 Subject: [PATCH 06/21] metrics-watcher: add Dockerfile --- services/metrics-watcher/Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 services/metrics-watcher/Dockerfile diff --git a/services/metrics-watcher/Dockerfile b/services/metrics-watcher/Dockerfile new file mode 100644 index 0000000..b8284ad --- /dev/null +++ b/services/metrics-watcher/Dockerfile @@ -0,0 +1,16 @@ +FROM node:22-alpine AS build +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY tsconfig.json ./ +COPY src ./src +RUN npm run build + +FROM node:22-alpine +WORKDIR /app +ENV NODE_ENV=production +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev +COPY --from=build /app/dist ./dist +EXPOSE 3300 +CMD ["node", "dist/server.js"] From 14885e9789cc46e8a709cb47003b401202134fac Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:21:23 +0530 Subject: [PATCH 07/21] docker-compose: add metrics-watcher service, isolate on observability-net --- docker-compose.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b782fbf..8c16c0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,6 +81,9 @@ services: - "9090:9090" depends_on: - checkout-api + networks: + - default + - observability-net grafana: image: grafana/grafana:11.2.0 @@ -164,8 +167,35 @@ services: networks: - sandbox-net + metrics-watcher: + build: ./services/metrics-watcher + container_name: chaos-metrics-watcher + environment: + PORT: "3300" + PROMETHEUS_URL: http://prometheus:9090 + ports: + - "127.0.0.1:3300:3300" + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:3300/health"] + interval: 5s + timeout: 5s + retries: 10 + restart: unless-stopped + depends_on: + - prometheus + # Isolated on its own network, separate from every other service's + # implicit default network — mirrors the sandbox-net fix M4's Qodo + # review taught us to apply. metrics-watcher's only legitimate call is + # to Prometheus; this keeps it unreachable from mcp-server's + # unauthenticated, destructive /mcp endpoint even if a compromised + # process inside it tried. Host access to its own published port + # (127.0.0.1:3300) is unaffected by which network it's attached to. + networks: + - observability-net + networks: sandbox-net: + observability-net: volumes: pg_primary_data: From 3831d973e01cb257ddf8e8ec2605c3e3cc05cc81 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:25:33 +0530 Subject: [PATCH 08/21] add M5 acceptance test --- scripts/verify-m5.sh | 102 ++++++++++++++++++++++++ services/metrics-watcher/src/cliCall.ts | 30 +++++++ 2 files changed, 132 insertions(+) create mode 100644 scripts/verify-m5.sh create mode 100644 services/metrics-watcher/src/cliCall.ts diff --git a/scripts/verify-m5.sh b/scripts/verify-m5.sh new file mode 100644 index 0000000..6547728 --- /dev/null +++ b/scripts/verify-m5.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# M5 acceptance test. Run from the repo root inside WSL: +# bash scripts/verify-m5.sh +# +# Verifies observe_impact reports accurate real-world severity against +# live Prometheus data: "none" at baseline, "hard" with a "matched" +# verdict during a real fault, correctly rejects invalid input, and +# confirms metrics-watcher cannot reach mcp-server over the network (the +# same isolation guard M4 added after Qodo caught the equivalent gap for +# blast-radius-sandbox). +set -euo pipefail + +API_URL="http://localhost:3000" +MCP_URL="http://localhost:3100" +METRICS_WATCHER_URL="http://localhost:3300" + +call_mcp() { + local args="${2:-}" + [ -n "$args" ] || args='{}' + docker compose exec -T mcp-server node dist/cliCall.js "$1" "$args" +} + +call_metrics_watcher() { + local args="${2:-}" + [ -n "$args" ] || args='{}' + docker compose exec -T metrics-watcher node dist/cliCall.js "$1" "$args" +} + +wait_for() { + local desc="$1" cmd="$2" tries="${3:-30}" + for _ in $(seq 1 "$tries"); do + if eval "$cmd" >/dev/null 2>&1; then + echo "OK: $desc" + return 0 + fi + sleep 2 + done + echo "FAIL: timed out waiting for $desc" + exit 1 +} + +echo "== bringing up stack ==" +docker compose up -d --build + +wait_for "checkout-api healthy" "curl -sf ${API_URL}/health" +wait_for "mcp-server healthy" "curl -sf ${MCP_URL}/health" +wait_for "metrics-watcher healthy" "curl -sf ${METRICS_WATCHER_URL}/health" + +echo "== settling for baseline traffic (60s) ==" +sleep 60 + +echo "== observe_impact at baseline: expect observedSeverity none ==" +baseline=$(call_metrics_watcher observe_impact '{"predicted_severity":"degraded","affected_routes":["/products"],"window_seconds":60}') +echo "$baseline" +printf '%s' "$baseline" | python3 -c " +import json, sys +result = json.load(sys.stdin) +observation = json.loads(result['content'][0]['text']) +assert observation['observedSeverity'] == 'none', f\"expected none, got {observation['observedSeverity']}\" +print('OK: baseline observed severity is none') +" + +echo "== pause_container(chaos-pg-replica, 90s) via MCP ==" +call_mcp pause_container '{"container":"chaos-pg-replica","duration_seconds":90}' +# Sleep a full minute so the 60s query window sits entirely inside the +# fault period (same lesson as verify-m1.sh/verify-m2.sh: a partial window +# dilutes the error rate below the assertion threshold even when the fault +# is working correctly). +sleep 60 + +echo "== observe_impact during fault: expect observedSeverity hard, verdict matched ==" +faulted=$(call_metrics_watcher observe_impact '{"predicted_severity":"hard","affected_routes":["/products"],"window_seconds":60}') +echo "$faulted" +printf '%s' "$faulted" | python3 -c " +import json, sys +result = json.load(sys.stdin) +observation = json.loads(result['content'][0]['text']) +assert observation['observedSeverity'] == 'hard', f\"expected hard, got {observation['observedSeverity']}\" +assert observation['verdict'] == 'matched', f\"expected matched, got {observation['verdict']}\" +print('OK: fault-window observed severity is hard, verdict matched') +" + +echo "== waiting for auto-revert (duration_seconds=90 must elapse on its own) ==" +sleep 90 + +echo "== observe_impact rejects empty affected_routes ==" +set +e +rejected=$(call_metrics_watcher observe_impact '{"predicted_severity":"hard","affected_routes":[],"window_seconds":60}' 2>&1) +rc=$? +set -e +[ "$rc" -ne 0 ] || { echo "FAIL: expected empty affected_routes to be rejected"; exit 1; } +echo "OK: empty affected_routes rejected ($rejected)" + +echo "== network isolation: metrics-watcher cannot reach mcp-server ==" +set +e +docker compose exec -T metrics-watcher wget -q --timeout=5 --spider http://mcp-server:3100/health +rc=$? +set -e +[ "$rc" -ne 0 ] || { echo "FAIL: metrics-watcher could reach mcp-server — network isolation broken"; exit 1; } +echo "OK: metrics-watcher cannot reach mcp-server" + +echo "== M5 ACCEPTANCE TEST PASSED ==" diff --git a/services/metrics-watcher/src/cliCall.ts b/services/metrics-watcher/src/cliCall.ts new file mode 100644 index 0000000..ecf59e2 --- /dev/null +++ b/services/metrics-watcher/src/cliCall.ts @@ -0,0 +1,30 @@ +// services/metrics-watcher/src/cliCall.ts +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; + +async function main() { + const [, , toolName, argsJson] = process.argv; + if (!toolName) { + console.error("usage: cliCall.js [argsJson]"); + process.exit(1); + } + const args = argsJson ? JSON.parse(argsJson) : {}; + + const client = new Client({ name: "verify-m5-cli", version: "1.0.0" }); + const transport = new StreamableHTTPClientTransport(new URL("http://localhost:3300/mcp")); + try { + await client.connect(transport); + const result = await client.callTool({ name: toolName, arguments: args }); + console.log(JSON.stringify(result)); + if (result.isError) { + process.exitCode = 1; + } + } finally { + await client.close(); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From 6498f8511466e4c11cbfddb21a32400f2c86ab0c Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:28:29 +0530 Subject: [PATCH 09/21] fix: mark scripts/verify-m5.sh executable (100755) Task 8 brief specified chmod +x scripts/verify-m5.sh, but the file was committed with mode 100644. core.fileMode=false in this repo means a filesystem chmod alone doesn't update the git index, so this uses git update-index --chmod=+x to fix the tracked mode. --- scripts/verify-m5.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/verify-m5.sh diff --git a/scripts/verify-m5.sh b/scripts/verify-m5.sh old mode 100644 new mode 100755 From 02d8f3e8ca422b7d38a379b2de43cf9f601f9077 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:31:46 +0530 Subject: [PATCH 10/21] agent: wire in the metrics-watcher connector --- agent/README.md | 43 +++++++++++++++++++++++++++++++++++++++-- agent/chaos-notary.json | 7 ++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/agent/README.md b/agent/README.md index 56d997d..88ed485 100644 --- a/agent/README.md +++ b/agent/README.md @@ -45,6 +45,20 @@ can't automate. **Do not widen the `127.0.0.1:3200:3200` binding either.** +- The same TrueForge instance also needs network access to the M5 + `metrics-watcher` service, for the same reachability reasons — + `docker-compose.yml` publishes it as `127.0.0.1:3300:3300`, loopback-only. + Its `/mcp` endpoint has no authentication either; the stakes are lower + since `observe_impact` is a read-only Prometheus query with no Docker + access, but the same reachability pattern applies. If TrueForge runs as + a process on the same host as the Docker stack, it can reach the server + at `http://localhost:3300`. If it can't, put TrueForge on this project's + Docker network (as above) and reach it via its compose service DNS name, + `http://chaos-metrics-watcher:3300`, rather than widening the published + host port. + + **Do not widen the `127.0.0.1:3300:3300` binding either.** + ## Setup 1. **Register `mcp-server` as a Connector.** In TrueForge's UI, go to @@ -82,7 +96,18 @@ can't automate. `require_approval_for_tools`: `predict_blast_radius` is read-only computation with no Docker access, so nothing on it needs gating. -3. **Load the manifest.** Via TrueForge's Agent Playground (paste/import +3. **Register `metrics-watcher` as a Connector.** Same UI, add a third MCP + connector pointing at `http://localhost:3300/mcp` if TrueForge runs on + the same host as the Docker stack, or `http://chaos-metrics-watcher:3300/mcp` + if TrueForge is instead running on this project's Docker network (see + Prerequisites above — do not point it at a widened `3300:3300` + binding), and name it `metrics-watcher` — the manifest's + `mcp_servers[2].name` must match whatever you name it here. Like + `blast-radius-sandbox`, this entry has no `require_approval_for_tools`: + `observe_impact` is a read-only Prometheus query with no Docker access, + so nothing on it needs gating. + +4. **Load the manifest.** Via TrueForge's Agent Playground (paste/import `chaos-notary.json`) or its SDK (`type: "truefoundry-agent"` manifests can be defined in code and saved programmatically — see TrueForge's docs). Save it to the Agent Registry. @@ -131,7 +156,21 @@ a real TrueForge instance before considering M3 done: `list_targets` has no application-health signal to report (only Docker status and fault-registry state). Check the Grafana dashboard yourself for actual workload recovery. -8. Repeat step 2 asking it to target a container NOT on the allowlist +8. **Confirm the closing report reflects `observe_impact`'s real verdict, + not a restated prediction.** Let the same experiment run through to + completion (the agent should call `observe_impact` once Docker/fault + state is confirmed clean in step 7, passing the `severity` from its + earlier `predict_blast_radius` call as `predicted_severity`, the + affected routes with their HTTP verbs stripped as `affected_routes`, + and the fault's `duration_seconds` as `window_seconds`). Separately, + call `observe_impact` yourself with those same arguments for that same + window and compare: the agent's closing report should state the same + real `observedSeverity` and `verdict` your direct call returns, not + just a repeat of what `predict_blast_radius` predicted earlier. If the + agent skips calling `observe_impact`, or its closing report only + restates the prediction instead of a real observed outcome, the wiring + from Task 9 of the M5 plan isn't working. +9. Repeat step 2 asking it to target a container NOT on the allowlist (e.g. "pause chaos-mcp-server") — confirm the tool call is rejected and the agent reports that clearly rather than retrying or working around it. diff --git a/agent/chaos-notary.json b/agent/chaos-notary.json index 2f354ff..caddbb6 100644 --- a/agent/chaos-notary.json +++ b/agent/chaos-notary.json @@ -9,7 +9,7 @@ "temperature": 0.3 } }, - "instructions": "You are the Chaos Notary agent: you run bounded, reversible resilience\nexperiments against a live demo stack (the \"Chaos Notary target stack\") to\nreveal how it fails, and you always stop for explicit human approval\nbefore taking any action that could disrupt the system.\n\n## What you can do\n\nYou have access to 7 tools on the `mcp-server` connector, restricted to a\nfixed 5-container allowlist (chaos-pg-primary, chaos-pg-replica,\nchaos-checkout-api, chaos-prometheus, chaos-grafana):\n\n- `list_targets` — read-only. Reports each allowlisted container's current\n Docker state and whether a fault is currently active on it — not\n application health, only Docker/fault state. Always safe to call. Call\n it first before proposing any experiment, and again after one.\n- `pause_container`, `stop_container`, `kill_container` — freeze, stop, or\n kill a container's process for a bounded duration_seconds (5-300s),\n after which it automatically recovers on its own even if nothing calls\n you back.\n- `inject_latency`, `inject_packet_loss` — add network latency or packet\n loss to a container's traffic for the same bounded duration.\n- `clear_fault` — manually and immediately revert whatever fault is\n currently active on a container. Safe to call any time; it only ever\n restores safety, so it does not require approval.\n\nEvery fault-injection tool above (pause_container, stop_container,\nkill_container, inject_latency, inject_packet_loss) is destructive and\nrequires human approval before you're allowed to invoke it — you will be\nprompted automatically. Do not try to work around this or suggest the\nhuman disable it.\n\nYou also have access to one tool on the `blast-radius-sandbox` connector,\nread-only and side-effect-free — it never requires approval:\n\n- `predict_blast_radius(container, fault_kind, latency_ms?, jitter_ms?, percent?)` —\n given a container and a fault type, returns a static prediction:\n `severity` (\"hard\" or \"degraded\"), the list of `affected` targets and\n how each is affected, and the list of `unaffected` targets. It does not\n touch Docker or the target stack in any way — it's a lookup, not an\n experiment. Call it to ground your stated intent in this actual\n prediction, not your own general reasoning.\n\n## How to run an experiment\n\n1. Call `list_targets` first to confirm the target isn't already faulted.\n2. Call `predict_blast_radius` for the container and fault type you're\n considering. If the fault is `inject_latency` or `inject_packet_loss`,\n include `latency_ms`/`percent` matching the value you're about to\n propose — omitting them makes the prediction default to a milder\n 'degraded' severity that may not reflect what you're actually\n planning to run. If the proposed `inject_latency` experiment also\n uses jitter, include `jitter_ms` too — real jitter can push actual\n delays well past the hard-severity threshold even when the base\n `latency_ms` alone would not. Use its actual `severity`, `affected`, and\n `unaffected` fields — not your own general reasoning — to determine\n what to say next.\n3. State what you're about to do and why, in plain language, before\n proposing the tool call: which container, which fault type, how long,\n and what you expect to happen, based on that prediction (e.g.\n \"pausing chaos-pg-replica for 60s should make GET /products start\n returning 503s — predict_blast_radius reports it affected with\n severity 'hard', while POST /orders is unaffected\").\n4. Propose the fault-injection tool call. Wait for approval.\n5. After the call returns, tell the human where to watch for the effect:\n the Grafana dashboard at http://localhost:3001/d/chaos-notary\n (provisioned by the M1 target stack) shows the live error-rate panel.\n You do not have a tool to query Prometheus directly yet — reading\n metrics automatically is planned for a later milestone and isn't built.\n Don't claim to have observed impact you can't actually see.\n6. Once the fault's duration_seconds has elapsed, call `list_targets`\n again to confirm the container's Docker status is running with no\n active fault. list_targets has no application-health signal — for\n confirmation the workload itself recovered, point the human at the\n dashboard. If Docker/fault state isn't clean, say so plainly and call\n `clear_fault` — that call never needs approval.\n\n## What you must never do\n\n- Never call a fault-injection tool without first stating your intent and\n getting the approval prompt to fire.\n- Never target a container outside the fixed allowlist — the tools will\n reject it, but don't try anyway or suggest working around it.\n- Never claim you've verified an experiment's outcome by a means you don't\n actually have (e.g. don't claim to have \"checked the metrics\" or that a\n container is \"healthy\" — list_targets only tells you Docker/fault state;\n report exactly that, and point the human at the dashboard for anything\n beyond it).\n", + "instructions": "You are the Chaos Notary agent: you run bounded, reversible resilience\nexperiments against a live demo stack (the \"Chaos Notary target stack\") to\nreveal how it fails, and you always stop for explicit human approval\nbefore taking any action that could disrupt the system.\n\n## What you can do\n\nYou have access to 7 tools on the `mcp-server` connector, restricted to a\nfixed 5-container allowlist (chaos-pg-primary, chaos-pg-replica,\nchaos-checkout-api, chaos-prometheus, chaos-grafana):\n\n- `list_targets` — read-only. Reports each allowlisted container's current\n Docker state and whether a fault is currently active on it — not\n application health, only Docker/fault state. Always safe to call. Call\n it first before proposing any experiment, and again after one.\n- `pause_container`, `stop_container`, `kill_container` — freeze, stop, or\n kill a container's process for a bounded duration_seconds (5-300s),\n after which it automatically recovers on its own even if nothing calls\n you back.\n- `inject_latency`, `inject_packet_loss` — add network latency or packet\n loss to a container's traffic for the same bounded duration.\n- `clear_fault` — manually and immediately revert whatever fault is\n currently active on a container. Safe to call any time; it only ever\n restores safety, so it does not require approval.\n\nEvery fault-injection tool above (pause_container, stop_container,\nkill_container, inject_latency, inject_packet_loss) is destructive and\nrequires human approval before you're allowed to invoke it — you will be\nprompted automatically. Do not try to work around this or suggest the\nhuman disable it.\n\nYou also have access to one tool on the `blast-radius-sandbox` connector,\nread-only and side-effect-free — it never requires approval:\n\n- `predict_blast_radius(container, fault_kind, latency_ms?, jitter_ms?, percent?)` —\n given a container and a fault type, returns a static prediction:\n `severity` (\"hard\" or \"degraded\"), the list of `affected` targets and\n how each is affected, and the list of `unaffected` targets. It does not\n touch Docker or the target stack in any way — it's a lookup, not an\n experiment. Call it to ground your stated intent in this actual\n prediction, not your own general reasoning.\n\nYou also have access to one tool on the `metrics-watcher` connector, also\nread-only and side-effect-free — it never requires approval:\n\n- `observe_impact(predicted_severity, affected_routes, window_seconds)` —\n takes the `predicted_severity`, `affected_routes`, and `window_seconds`\n you give it and queries Prometheus for chaos-checkout-api's real error\n rate and latency over that window, returning `observedSeverity` and a\n `verdict` comparing what was actually observed against what was\n predicted. It reads Prometheus only — it does not touch Docker or the\n target stack in any way.\n\n## How to run an experiment\n\n1. Call `list_targets` first to confirm the target isn't already faulted.\n2. Call `predict_blast_radius` for the container and fault type you're\n considering. If the fault is `inject_latency` or `inject_packet_loss`,\n include `latency_ms`/`percent` matching the value you're about to\n propose — omitting them makes the prediction default to a milder\n 'degraded' severity that may not reflect what you're actually\n planning to run. If the proposed `inject_latency` experiment also\n uses jitter, include `jitter_ms` too — real jitter can push actual\n delays well past the hard-severity threshold even when the base\n `latency_ms` alone would not. Use its actual `severity`, `affected`, and\n `unaffected` fields — not your own general reasoning — to determine\n what to say next.\n3. State what you're about to do and why, in plain language, before\n proposing the tool call: which container, which fault type, how long,\n and what you expect to happen, based on that prediction (e.g.\n \"pausing chaos-pg-replica for 60s should make GET /products start\n returning 503s — predict_blast_radius reports it affected with\n severity 'hard', while POST /orders is unaffected\").\n4. Propose the fault-injection tool call. Wait for approval.\n5. After the call returns, tell the human where to watch for the effect:\n the Grafana dashboard at http://localhost:3001/d/chaos-notary\n (provisioned by the M1 target stack) shows the live error-rate panel.\n Once the fault has fully reverted, you will check the real impact\n yourself via `observe_impact` (step 7 below) — don't claim to have\n observed impact before then.\n6. Once the fault's duration_seconds has elapsed, call `list_targets`\n again to confirm the container's Docker status is running with no\n active fault. list_targets has no application-health signal — for\n confirmation the workload itself recovered, point the human at the\n dashboard. If Docker/fault state isn't clean, say so plainly and call\n `clear_fault` — that call never needs approval.\n7. Once Docker/fault state is confirmed clean, call `observe_impact`.\n This step is mandatory, not optional — exactly like `predict_blast_radius`\n in step 2, you must always call it, never skip it. Pass the `severity`\n from your earlier `predict_blast_radius` call as `predicted_severity`,\n its `affected` targets as `affected_routes` (stripping any leading HTTP\n verb from each — e.g. `\"GET /products\"` becomes `\"/products\"`), and the\n fault's `duration_seconds` as `window_seconds`. Report the tool's real\n `observedSeverity` and `verdict` to the human as the close of the\n experiment report — a real, checked outcome, not a restated prediction.\n\n## What you must never do\n\n- Never call a fault-injection tool without first stating your intent and\n getting the approval prompt to fire.\n- Never target a container outside the fixed allowlist — the tools will\n reject it, but don't try anyway or suggest working around it.\n- Never claim you've verified an experiment's outcome by a means you don't\n actually have. `list_targets` only tells you Docker/fault state — never\n claim it told you a container is \"healthy\"; report exactly that, and\n point the human at the dashboard for anything beyond it. `observe_impact`\n is different: its `observedSeverity`, `verdict`, and per-route numbers\n come from a real, current Prometheus query, so you may cite them\n directly once you have actually called it — never state them before\n calling it, and never state anything beyond what its actual return\n value contains.\n", "mcp_servers": [ { "type": "truefoundry-mcp-registry", @@ -27,6 +27,11 @@ "type": "truefoundry-mcp-registry", "name": "blast-radius-sandbox", "enable_tools": ["@all"] + }, + { + "type": "truefoundry-mcp-registry", + "name": "metrics-watcher", + "enable_tools": ["@all"] } ], "config": { From 4736a98f70a4481f90d205c479cdd09c2de812d3 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:35:29 +0530 Subject: [PATCH 11/21] README: document M5 metrics-watcher --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0fc5270..2d13788 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ sandbox for blast-radius computation. - **M2 — MCP server**: done. Exposes chaos tools (container pause/stop/kill, network latency/loss injection) over Streamable HTTP, wrapping Pumba, with bounded duration per fault and guaranteed auto-revert. - **M3 — Agent + approval gates**: done. TrueForge agent manifest declares human approval for every destructive chaos tool; whether the gate actually fires is confirmed by manually running it against a live TrueForge instance, a check not yet performed in this repo — see [`agent/README.md`](agent/README.md) for setup and that verification walkthrough. - **M4 — Blast-radius sandbox**: done. Sealed read-only service for predicting chaos experiment blast radius, wired as a second MCP connector in the agent manifest, with zero Docker access to ensure it cannot affect the live stack. -- M5 (metrics-watcher subagent), M6 (hardening) are not yet built. +- **M5 — Metrics-watcher**: done. Reports what Prometheus actually recorded after a chaos fault reverts and compares it against M4's prediction, wired as a third MCP connector in the agent manifest, closing the predict-then-observe loop. + +M6 (hardening) is not yet built. ## M1 — Target stack @@ -86,6 +88,17 @@ non-allowlisted container. bash scripts/verify-m4.sh ``` +`scripts/verify-m5.sh` exercises the metrics-watcher service in isolation (the full +stack is brought up), checking that baseline observed severity is "none," a real +pause-triggered fault produces "hard" observed severity with a "matched" verdict, +invalid input (empty affected_routes) is rejected, and metrics-watcher cannot reach +mcp-server over the network (the same isolation guard applied in M4 after Qodo caught +the gap). + +```bash +bash scripts/verify-m5.sh +``` + ### Services | Service | Container | Port | Role | @@ -98,6 +111,7 @@ bash scripts/verify-m4.sh | `loadgen` | `chaos-loadgen` | — | Fire-and-forget traffic generator, ~10rps, 70% reads / 30% writes | | `mcp-server` | `chaos-mcp-server` | 3100 | MCP server exposing chaos tools: pause/stop/kill containers, inject network latency/packet loss, with bounded duration and guaranteed auto-revert | | `blast-radius-sandbox` | `chaos-blast-radius-sandbox` | 3200 | Read-only prediction service: computes expected blast radius of chaos experiments using static topology model, no Docker access | +| `metrics-watcher` | `chaos-metrics-watcher` | 3300 | Observes and reports actual impact severity from live Prometheus data after chaos faults, wired as third MCP connector alongside MCP server and blast-radius-sandbox | All credentials in `docker-compose.yml` are dev-only placeholders (`chaos_dev_only_not_a_secret`, `chaos_dev_replica_not_a_secret`) — obviously From 20710a8659ed2c35c43fc6018377a7be7f4c4e1e Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:37:40 +0530 Subject: [PATCH 12/21] README: fix self-contradictory verify-m5.sh description --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2d13788..106802b 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,12 @@ non-allowlisted container. bash scripts/verify-m4.sh ``` -`scripts/verify-m5.sh` exercises the metrics-watcher service in isolation (the full -stack is brought up), checking that baseline observed severity is "none," a real -pause-triggered fault produces "hard" observed severity with a "matched" verdict, -invalid input (empty affected_routes) is rejected, and metrics-watcher cannot reach -mcp-server over the network (the same isolation guard applied in M4 after Qodo caught -the gap). +`scripts/verify-m5.sh` brings up the full stack and exercises the metrics-watcher +service, checking that baseline observed severity is "none," a real pause-triggered +fault produces "hard" observed severity with a "matched" verdict, invalid input +(empty affected_routes) is rejected, and verifying metrics-watcher's network isolation +from mcp-server — placed on its own `observability-net` to prevent reaching +mcp-server, the same topology-isolation guard M4 established via `sandbox-net`. ```bash bash scripts/verify-m5.sh From c3b377cfbf4b0bc3df13d5a67d2daec8a28b8838 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:56:43 +0530 Subject: [PATCH 13/21] fix: filter observe_impact to route-shaped targets and buffer its window (C1, C2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mandatory observe_impact workflow step reliably failed for 3 of 5 allowlisted containers: predict_blast_radius's `affected` list includes non-route targets (Grafana dashboard descriptions, Prometheus scrape/query descriptions) that don't strip down to a known route, and zod's fixed route enum (kept as-is, deliberately) rejects the whole call outright. agent/chaos-notary.json's workflow now filters `affected` down to route-shaped targets before calling observe_impact, and skips the call entirely (with a plain explanation) when no route-shaped target remains. Separately, observe_impact had no time anchor: a window_seconds equal to just the fault's duration_seconds risks landing entirely in the already-recovered period once real reasoning/tool-call delay is accounted for. The workflow now asks for duration_seconds + ~60s of buffer instead. scripts/verify-m5.sh gains a regression assertion that calls observe_impact well after the fault has reverted, with a buffered window, and asserts it still reports "hard"/"matched" — the test that would have caught this. Bundled in the same files: the observe_impact tool description and the agent's own capability bullet now note observedSeverity/verdict are error-rate-only (I3), and verify-m5.sh gains a positive-control wget check before its network-isolation negative control so that assertion can't false-pass on a broken container/exec rather than real isolation (I4). agent/README.md's manual verification walkthrough is updated to match. --- agent/README.md | 38 +++++++++++++++++++-------- agent/chaos-notary.json | 2 +- scripts/verify-m5.sh | 25 ++++++++++++++++++ services/metrics-watcher/src/tools.ts | 2 +- 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/agent/README.md b/agent/README.md index 88ed485..aaef7ac 100644 --- a/agent/README.md +++ b/agent/README.md @@ -158,18 +158,34 @@ a real TrueForge instance before considering M3 done: for actual workload recovery. 8. **Confirm the closing report reflects `observe_impact`'s real verdict, not a restated prediction.** Let the same experiment run through to - completion (the agent should call `observe_impact` once Docker/fault - state is confirmed clean in step 7, passing the `severity` from its - earlier `predict_blast_radius` call as `predicted_severity`, the - affected routes with their HTTP verbs stripped as `affected_routes`, - and the fault's `duration_seconds` as `window_seconds`). Separately, - call `observe_impact` yourself with those same arguments for that same - window and compare: the agent's closing report should state the same - real `observedSeverity` and `verdict` your direct call returns, not + completion. For `chaos-pg-replica` + `pause`, `predict_blast_radius`'s + only `affected` target is `GET /products`, which strips down to the + known route `/products`, so `observe_impact` applies here and the agent + should call it once Docker/fault state is confirmed clean in step 7: + passing the `severity` from its earlier `predict_blast_radius` call as + `predicted_severity`, `["/products"]` as `affected_routes`, and + `window_seconds` set to the fault's `duration_seconds` **plus roughly 60 + seconds of buffer** (e.g. for this 30-second pause, expect something + around 90, not exactly 30 — a window that ends exactly at + `duration_seconds` risks landing entirely in the already-recovered + period once the seconds of reasoning/tool-call time between the fault + reverting and this call are accounted for). Separately, call + `observe_impact` yourself with those same arguments (predicted_severity + `"hard"`, affected_routes `["/products"]`, and a similarly buffered + window_seconds) and compare: the agent's closing report should state the + same real `observedSeverity` and `verdict` your direct call returns, not just a repeat of what `predict_blast_radius` predicted earlier. If the - agent skips calling `observe_impact`, or its closing report only - restates the prediction instead of a real observed outcome, the wiring - from Task 9 of the M5 plan isn't working. + agent skips calling `observe_impact` when it does apply, passes a + `window_seconds` with no buffer, or its closing report only restates the + prediction instead of a real observed outcome, the wiring from Task 9 of + the M5 plan (and the C1/C2 fixes on top of it) isn't working. + + Not every fault reaches this branch. If you instead run an experiment + whose `predict_blast_radius affected` list contains no route-shaped + targets (e.g. pausing `chaos-grafana`, or `inject_packet_loss` at + `percent: 0`), confirm the agent does NOT call `observe_impact` at all, + and instead says plainly that no route-level observation applies to + that fault. 9. Repeat step 2 asking it to target a container NOT on the allowlist (e.g. "pause chaos-mcp-server") — confirm the tool call is rejected and the agent reports that clearly rather than retrying or working around diff --git a/agent/chaos-notary.json b/agent/chaos-notary.json index caddbb6..56797cd 100644 --- a/agent/chaos-notary.json +++ b/agent/chaos-notary.json @@ -9,7 +9,7 @@ "temperature": 0.3 } }, - "instructions": "You are the Chaos Notary agent: you run bounded, reversible resilience\nexperiments against a live demo stack (the \"Chaos Notary target stack\") to\nreveal how it fails, and you always stop for explicit human approval\nbefore taking any action that could disrupt the system.\n\n## What you can do\n\nYou have access to 7 tools on the `mcp-server` connector, restricted to a\nfixed 5-container allowlist (chaos-pg-primary, chaos-pg-replica,\nchaos-checkout-api, chaos-prometheus, chaos-grafana):\n\n- `list_targets` — read-only. Reports each allowlisted container's current\n Docker state and whether a fault is currently active on it — not\n application health, only Docker/fault state. Always safe to call. Call\n it first before proposing any experiment, and again after one.\n- `pause_container`, `stop_container`, `kill_container` — freeze, stop, or\n kill a container's process for a bounded duration_seconds (5-300s),\n after which it automatically recovers on its own even if nothing calls\n you back.\n- `inject_latency`, `inject_packet_loss` — add network latency or packet\n loss to a container's traffic for the same bounded duration.\n- `clear_fault` — manually and immediately revert whatever fault is\n currently active on a container. Safe to call any time; it only ever\n restores safety, so it does not require approval.\n\nEvery fault-injection tool above (pause_container, stop_container,\nkill_container, inject_latency, inject_packet_loss) is destructive and\nrequires human approval before you're allowed to invoke it — you will be\nprompted automatically. Do not try to work around this or suggest the\nhuman disable it.\n\nYou also have access to one tool on the `blast-radius-sandbox` connector,\nread-only and side-effect-free — it never requires approval:\n\n- `predict_blast_radius(container, fault_kind, latency_ms?, jitter_ms?, percent?)` —\n given a container and a fault type, returns a static prediction:\n `severity` (\"hard\" or \"degraded\"), the list of `affected` targets and\n how each is affected, and the list of `unaffected` targets. It does not\n touch Docker or the target stack in any way — it's a lookup, not an\n experiment. Call it to ground your stated intent in this actual\n prediction, not your own general reasoning.\n\nYou also have access to one tool on the `metrics-watcher` connector, also\nread-only and side-effect-free — it never requires approval:\n\n- `observe_impact(predicted_severity, affected_routes, window_seconds)` —\n takes the `predicted_severity`, `affected_routes`, and `window_seconds`\n you give it and queries Prometheus for chaos-checkout-api's real error\n rate and latency over that window, returning `observedSeverity` and a\n `verdict` comparing what was actually observed against what was\n predicted. It reads Prometheus only — it does not touch Docker or the\n target stack in any way.\n\n## How to run an experiment\n\n1. Call `list_targets` first to confirm the target isn't already faulted.\n2. Call `predict_blast_radius` for the container and fault type you're\n considering. If the fault is `inject_latency` or `inject_packet_loss`,\n include `latency_ms`/`percent` matching the value you're about to\n propose — omitting them makes the prediction default to a milder\n 'degraded' severity that may not reflect what you're actually\n planning to run. If the proposed `inject_latency` experiment also\n uses jitter, include `jitter_ms` too — real jitter can push actual\n delays well past the hard-severity threshold even when the base\n `latency_ms` alone would not. Use its actual `severity`, `affected`, and\n `unaffected` fields — not your own general reasoning — to determine\n what to say next.\n3. State what you're about to do and why, in plain language, before\n proposing the tool call: which container, which fault type, how long,\n and what you expect to happen, based on that prediction (e.g.\n \"pausing chaos-pg-replica for 60s should make GET /products start\n returning 503s — predict_blast_radius reports it affected with\n severity 'hard', while POST /orders is unaffected\").\n4. Propose the fault-injection tool call. Wait for approval.\n5. After the call returns, tell the human where to watch for the effect:\n the Grafana dashboard at http://localhost:3001/d/chaos-notary\n (provisioned by the M1 target stack) shows the live error-rate panel.\n Once the fault has fully reverted, you will check the real impact\n yourself via `observe_impact` (step 7 below) — don't claim to have\n observed impact before then.\n6. Once the fault's duration_seconds has elapsed, call `list_targets`\n again to confirm the container's Docker status is running with no\n active fault. list_targets has no application-health signal — for\n confirmation the workload itself recovered, point the human at the\n dashboard. If Docker/fault state isn't clean, say so plainly and call\n `clear_fault` — that call never needs approval.\n7. Once Docker/fault state is confirmed clean, call `observe_impact`.\n This step is mandatory, not optional — exactly like `predict_blast_radius`\n in step 2, you must always call it, never skip it. Pass the `severity`\n from your earlier `predict_blast_radius` call as `predicted_severity`,\n its `affected` targets as `affected_routes` (stripping any leading HTTP\n verb from each — e.g. `\"GET /products\"` becomes `\"/products\"`), and the\n fault's `duration_seconds` as `window_seconds`. Report the tool's real\n `observedSeverity` and `verdict` to the human as the close of the\n experiment report — a real, checked outcome, not a restated prediction.\n\n## What you must never do\n\n- Never call a fault-injection tool without first stating your intent and\n getting the approval prompt to fire.\n- Never target a container outside the fixed allowlist — the tools will\n reject it, but don't try anyway or suggest working around it.\n- Never claim you've verified an experiment's outcome by a means you don't\n actually have. `list_targets` only tells you Docker/fault state — never\n claim it told you a container is \"healthy\"; report exactly that, and\n point the human at the dashboard for anything beyond it. `observe_impact`\n is different: its `observedSeverity`, `verdict`, and per-route numbers\n come from a real, current Prometheus query, so you may cite them\n directly once you have actually called it — never state them before\n calling it, and never state anything beyond what its actual return\n value contains.\n", + "instructions": "You are the Chaos Notary agent: you run bounded, reversible resilience\nexperiments against a live demo stack (the \"Chaos Notary target stack\") to\nreveal how it fails, and you always stop for explicit human approval\nbefore taking any action that could disrupt the system.\n\n## What you can do\n\nYou have access to 7 tools on the `mcp-server` connector, restricted to a\nfixed 5-container allowlist (chaos-pg-primary, chaos-pg-replica,\nchaos-checkout-api, chaos-prometheus, chaos-grafana):\n\n- `list_targets` — read-only. Reports each allowlisted container's current\n Docker state and whether a fault is currently active on it — not\n application health, only Docker/fault state. Always safe to call. Call\n it first before proposing any experiment, and again after one.\n- `pause_container`, `stop_container`, `kill_container` — freeze, stop, or\n kill a container's process for a bounded duration_seconds (5-300s),\n after which it automatically recovers on its own even if nothing calls\n you back.\n- `inject_latency`, `inject_packet_loss` — add network latency or packet\n loss to a container's traffic for the same bounded duration.\n- `clear_fault` — manually and immediately revert whatever fault is\n currently active on a container. Safe to call any time; it only ever\n restores safety, so it does not require approval.\n\nEvery fault-injection tool above (pause_container, stop_container,\nkill_container, inject_latency, inject_packet_loss) is destructive and\nrequires human approval before you're allowed to invoke it — you will be\nprompted automatically. Do not try to work around this or suggest the\nhuman disable it.\n\nYou also have access to one tool on the `blast-radius-sandbox` connector,\nread-only and side-effect-free — it never requires approval:\n\n- `predict_blast_radius(container, fault_kind, latency_ms?, jitter_ms?, percent?)` —\n given a container and a fault type, returns a static prediction:\n `severity` (\"hard\" or \"degraded\"), the list of `affected` targets and\n how each is affected, and the list of `unaffected` targets. It does not\n touch Docker or the target stack in any way — it's a lookup, not an\n experiment. Call it to ground your stated intent in this actual\n prediction, not your own general reasoning.\n\nYou also have access to one tool on the `metrics-watcher` connector, also\nread-only and side-effect-free — it never requires approval:\n\n- `observe_impact(predicted_severity, affected_routes, window_seconds)` —\n takes the `predicted_severity`, `affected_routes`, and `window_seconds`\n you give it and queries Prometheus for chaos-checkout-api's real error\n rate and latency over that window, returning `observedSeverity` and a\n `verdict` comparing what was actually observed against what was\n predicted. It reads Prometheus only — it does not touch Docker or the\n target stack in any way. `observedSeverity` and `verdict` are derived\n from error rate only — for a latency-only fault with no accompanying\n error spike, consult the response's per-route `avgLatencyMs` directly\n rather than relying on the verdict alone.\n\n## How to run an experiment\n\n1. Call `list_targets` first to confirm the target isn't already faulted.\n2. Call `predict_blast_radius` for the container and fault type you're\n considering. If the fault is `inject_latency` or `inject_packet_loss`,\n include `latency_ms`/`percent` matching the value you're about to\n propose — omitting them makes the prediction default to a milder\n 'degraded' severity that may not reflect what you're actually\n planning to run. If the proposed `inject_latency` experiment also\n uses jitter, include `jitter_ms` too — real jitter can push actual\n delays well past the hard-severity threshold even when the base\n `latency_ms` alone would not. Use its actual `severity`, `affected`, and\n `unaffected` fields — not your own general reasoning — to determine\n what to say next.\n3. State what you're about to do and why, in plain language, before\n proposing the tool call: which container, which fault type, how long,\n and what you expect to happen, based on that prediction (e.g.\n \"pausing chaos-pg-replica for 60s should make GET /products start\n returning 503s — predict_blast_radius reports it affected with\n severity 'hard', while POST /orders is unaffected\").\n4. Propose the fault-injection tool call. Wait for approval.\n5. After the call returns, tell the human where to watch for the effect:\n the Grafana dashboard at http://localhost:3001/d/chaos-notary\n (provisioned by the M1 target stack) shows the live error-rate panel.\n Once the fault has fully reverted, you will check the real impact\n yourself via `observe_impact` (step 7 below) — don't claim to have\n observed impact before then.\n6. Once the fault's duration_seconds has elapsed, call `list_targets`\n again to confirm the container's Docker status is running with no\n active fault. list_targets has no application-health signal — for\n confirmation the workload itself recovered, point the human at the\n dashboard. If Docker/fault state isn't clean, say so plainly and call\n `clear_fault` — that call never needs approval.\n7. Once Docker/fault state is confirmed clean, check whether `observe_impact`\n applies to this fault. Take `predict_blast_radius`'s `affected` targets\n and filter them down to only the entries that are actual checkout-api\n route paths: strip any leading HTTP verb from each (e.g. `\"GET /products\"`\n becomes `\"/products\"`), then keep only the ones that then exactly match\n one of the 4 known routes (`/products`, `/orders`, `/health`,\n `/metrics`). Some `affected` targets are NOT routes at all — e.g.\n `\"chaos-prometheus's scrape of /metrics\"`, `\"Grafana dashboard\n (chaos-notary)\"`, `\"Prometheus query API (:9090)\"`, or `\"Grafana\n dashboard UI\"` describe impact on Prometheus or Grafana themselves, not\n on a checkout-api route. Exclude these — never force-map a non-route\n target onto a route just because it looks similar.\n - If the filtered list has one or more routes in it, call\n `observe_impact`. This is mandatory whenever it applies — never skip\n it. Pass the `severity` from your earlier `predict_blast_radius` call\n as `predicted_severity`, the filtered route list as `affected_routes`,\n and `window_seconds` set to the fault's `duration_seconds` plus\n roughly 60 seconds of buffer (e.g. a 30-second fault should pass\n around 90, not exactly 30). The buffer matters: some amount of\n reasoning and tool-call time always passes between the fault reverting\n and this call being made, and you have no reliable way to measure that\n elapsed time precisely — a window that's exactly `duration_seconds`\n risks landing entirely in the already-recovered period and\n under-reporting severity, while a buffered window still reaches back\n far enough to cover the fault's actual active period. Report the\n tool's real `observedSeverity` and `verdict` to the human as the close\n of the experiment report — a real, checked outcome, not a restated\n prediction.\n - If the filtered list is empty — e.g. the fault's only predicted impact\n is on Grafana's dashboard, Prometheus's own query API, or it was a\n 0%-magnitude network fault with an empty `affected` list — do not call\n `observe_impact`. State plainly in your report that no route-level\n observation applies to this particular fault, since `observe_impact`\n only has visibility into chaos-checkout-api's own request-route\n metrics, not into Prometheus's or Grafana's own health.\n\n## What you must never do\n\n- Never call a fault-injection tool without first stating your intent and\n getting the approval prompt to fire.\n- Never target a container outside the fixed allowlist — the tools will\n reject it, but don't try anyway or suggest working around it.\n- Never claim you've verified an experiment's outcome by a means you don't\n actually have. `list_targets` only tells you Docker/fault state — never\n claim it told you a container is \"healthy\"; report exactly that, and\n point the human at the dashboard for anything beyond it. `observe_impact`\n is different: its `observedSeverity`, `verdict`, and per-route numbers\n come from a real, current Prometheus query, so you may cite them\n directly once you have actually called it — never state them before\n calling it, and never state anything beyond what its actual return\n value contains.\n", "mcp_servers": [ { "type": "truefoundry-mcp-registry", diff --git a/scripts/verify-m5.sh b/scripts/verify-m5.sh index 6547728..06adff4 100755 --- a/scripts/verify-m5.sh +++ b/scripts/verify-m5.sh @@ -91,6 +91,31 @@ set -e [ "$rc" -ne 0 ] || { echo "FAIL: expected empty affected_routes to be rejected"; exit 1; } echo "OK: empty affected_routes rejected ($rejected)" +echo "== observe_impact called well after revert, with a buffered window: still reports hard/matched (C2 regression) ==" +# This is the exact scenario C2 broke: by now the fault (duration_seconds=90) +# reverted a good while ago and more script time has elapsed since. A window +# equal to just the fault's duration would land entirely in the recovered +# period and report "none"/"milder_than_predicted". A buffered window +# (duration_seconds + 60) still reaches back far enough to cover the fault's +# active period despite the delay. +delayed=$(call_metrics_watcher observe_impact '{"predicted_severity":"hard","affected_routes":["/products"],"window_seconds":150}') +echo "$delayed" +printf '%s' "$delayed" | python3 -c " +import json, sys +result = json.load(sys.stdin) +observation = json.loads(result['content'][0]['text']) +assert observation['observedSeverity'] == 'hard', f\"expected hard, got {observation['observedSeverity']}\" +assert observation['verdict'] == 'matched', f\"expected matched, got {observation['verdict']}\" +print('OK: buffered post-revert-delay observe_impact still reports hard/matched') +" + +echo "== network isolation positive control: metrics-watcher CAN reach prometheus ==" +# Proves wget works inside the metrics-watcher image and the container can +# reach a legitimate target, so the negative-control failure below is +# attributable to network topology, not a broken wget/container/exec. +docker compose exec -T metrics-watcher wget -q --timeout=5 --spider http://prometheus:9090/-/healthy +echo "OK: metrics-watcher can reach prometheus" + echo "== network isolation: metrics-watcher cannot reach mcp-server ==" set +e docker compose exec -T metrics-watcher wget -q --timeout=5 --spider http://mcp-server:3100/health diff --git a/services/metrics-watcher/src/tools.ts b/services/metrics-watcher/src/tools.ts index d45e137..25ce7e4 100644 --- a/services/metrics-watcher/src/tools.ts +++ b/services/metrics-watcher/src/tools.ts @@ -59,7 +59,7 @@ export function registerTools( { title: "Observe real chaos-experiment impact", description: - "Query Prometheus for checkout-api's real, currently-measured error rate and latency per route, and compare against a predicted severity from predict_blast_radius. Call this after a fault reverts, passing the same predicted_severity and affected routes you got from predict_blast_radius (stripped of any leading HTTP verb, e.g. 'GET /products' becomes '/products'), and the fault's duration_seconds as window_seconds.", + "Query Prometheus for checkout-api's real, currently-measured error rate and latency per route, and compare against a predicted severity from predict_blast_radius. Call this after a fault reverts, passing the same predicted_severity and affected routes you got from predict_blast_radius (stripped of any leading HTTP verb, e.g. 'GET /products' becomes '/products'; only pass targets that are actual routes — some predict_blast_radius targets describe impact on Prometheus/Grafana themselves and are not routes at all, so skip calling this tool if none of the affected targets are routes). Pass window_seconds with some margin beyond the fault's own duration_seconds (e.g. duration_seconds + ~60s) to account for the delay between the fault ending and this call being made — a window equal to exactly duration_seconds risks landing entirely in the already-recovered period. Note observedSeverity/verdict reflect error rate only; for a latency-only fault with no accompanying error spike, consult the response's per-route avgLatencyMs directly rather than relying on the verdict alone.", inputSchema: { predicted_severity: z.enum(["hard", "degraded"]), affected_routes: z.array(z.enum(KNOWN_ROUTES)).min(1), From ad59eb1b6b813bdababef1f0ba8d41cf38470f4e Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:56:50 +0530 Subject: [PATCH 14/21] fix: correct latency aggregation, add fetch timeout, guard non-finite values (I1, I2, M3) - Wrap the avgLatencyMs PromQL query's numerator and denominator in sum(), matching the error-rate/request-count queries. http_request_duration_seconds is labeled by both route and status, so during a fault the un-summed query returned multiple series and result[0] picked an arbitrary status code's latency rather than an aggregate (I1). - Add a 5s AbortSignal.timeout to the Prometheus fetch call, mirroring verify-m1.sh's curl --max-time 5, so a paused chaos-prometheus (an allowlisted target) can't hang the tool indefinitely (I2). Verified the test fixtures' single-argument fake fetchers remain structurally compatible with typeof fetch under strict mode; no test changes needed. - Guard against Prometheus returning non-finite value strings ("NaN", "+Inf"): check Number.isFinite before returning, falling back to the same null "no data" semantics instead of letting a NaN silently become a contradictory "degraded" classification downstream (M3). --- services/metrics-watcher/src/prometheus.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/metrics-watcher/src/prometheus.ts b/services/metrics-watcher/src/prometheus.ts index 818fd62..4eb18be 100644 --- a/services/metrics-watcher/src/prometheus.ts +++ b/services/metrics-watcher/src/prometheus.ts @@ -15,7 +15,7 @@ interface PrometheusQueryResult { async function query(baseUrl: string, promql: string, fetchImpl: typeof fetch): Promise { const url = new URL("/api/v1/query", baseUrl); url.searchParams.set("query", promql); - const res = await fetchImpl(url.toString()); + const res = await fetchImpl(url.toString(), { signal: AbortSignal.timeout(5000) }); if (!res.ok) { throw new Error(`Prometheus query failed: ${res.status} ${res.statusText}`); } @@ -25,7 +25,11 @@ async function query(baseUrl: string, promql: string, fetchImpl: typeof fetch): } const result = body.data?.result ?? []; if (result.length === 0) return null; - return Number(result[0].value[1]); + const n = Number(result[0].value[1]); + // Prometheus can legitimately return "NaN" or "+Inf"/"-Inf" as value + // strings; treat those the same as "no data" rather than letting a + // non-finite number leak into severity classification downstream. + return Number.isFinite(n) ? n : null; } export async function queryRouteMetrics( @@ -52,7 +56,7 @@ export async function queryRouteMetrics( const avgLatencySeconds = await query( baseUrl, - `rate(http_request_duration_seconds_sum{route="${route}"}[${windowSeconds}s]) / rate(http_request_duration_seconds_count{route="${route}"}[${windowSeconds}s])`, + `sum(rate(http_request_duration_seconds_sum{route="${route}"}[${windowSeconds}s])) / sum(rate(http_request_duration_seconds_count{route="${route}"}[${windowSeconds}s]))`, fetchImpl, ); From 0aee5674fe9f8305cd8eb28ff4de1428f779dd4f Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:56:56 +0530 Subject: [PATCH 15/21] test: make observe_impact's fake fetcher route-aware (I5) The fake fetcher in tools.test.ts ignored its `route` argument and always returned the same metrics object, so no test could actually verify that observedSeverity is scoped to affected_routes only, or that the response's routes field reports all 4 known routes' real metrics rather than just the affected ones. Replaced it with a dispatch-on-route helper and added: - a scoping test (an unrelated route failing hard must not affect the verdict for a different, healthy affected_routes selection) - an all-4-routes test (routes field always has real per-route data) - a worse_than_predicted end-to-end case - an explicit error-propagation test through the real registerTools/McpServer/Client round trip Test count in this file: 2 -> 6 (net +4, matching the fix wave's target of 15 -> ~19 across the whole suite). --- services/metrics-watcher/test/tools.test.ts | 87 +++++++++++++++++++-- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/services/metrics-watcher/test/tools.test.ts b/services/metrics-watcher/test/tools.test.ts index 049a56f..2cf1ead 100644 --- a/services/metrics-watcher/test/tools.test.ts +++ b/services/metrics-watcher/test/tools.test.ts @@ -4,23 +4,98 @@ import assert from "node:assert/strict"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; -import { handleObserveImpact, registerTools } from "../src/tools.js"; +import { handleObserveImpact, registerTools, KNOWN_ROUTES, type KnownRoute } from "../src/tools.js"; +import type { RouteMetrics } from "../src/prometheus.js"; + +// A route-aware fake fetcher: unlike a fetcher that ignores its `route` +// argument, this one dispatches on it, so tests can prove observe_impact's +// severity computation is actually scoped to affected_routes and that the +// `routes` field in the response reflects each route's own real metrics. +const BASELINE: RouteMetrics = { errorRatePercent: 0, avgLatencyMs: 5, requestCount: 100 }; + +function makeFetcher(overrides: Partial>) { + return async (route: KnownRoute): Promise => overrides[route] ?? BASELINE; +} test("handleObserveImpact reports hard severity and a matched verdict when the affected route is failing hard", async () => { const result = await handleObserveImpact( { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 60 }, - async () => ({ + makeFetcher({ "/products": { errorRatePercent: 87, avgLatencyMs: 5, requestCount: 20 }, - "/orders": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, - "/health": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, - "/metrics": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, - })["/products"], + }), ); assert.equal(result.observedSeverity, "hard"); assert.equal(result.verdict, "matched"); assert.equal(result.routes["/products"].errorRatePercent, 87); }); +test("handleObserveImpact scopes observedSeverity to affected_routes only, ignoring unrelated routes' severity", async () => { + const result = await handleObserveImpact( + { predicted_severity: "hard", affected_routes: ["/orders"], window_seconds: 60 }, + makeFetcher({ + // /products is failing hard, but it's NOT in affected_routes. + "/products": { errorRatePercent: 90, avgLatencyMs: 5, requestCount: 50 }, + // /orders — the only affected route — has no errors at all. + "/orders": { errorRatePercent: 0, avgLatencyMs: 5, requestCount: 50 }, + }), + ); + assert.equal(result.observedSeverity, "none"); + assert.equal(result.verdict, "milder_than_predicted"); + // The unrelated route's real (hard) severity must still be visible in the + // full routes breakdown — it's just not allowed to influence the verdict. + assert.equal(result.routes["/products"].errorRatePercent, 90); +}); + +test("handleObserveImpact's routes field always contains all 4 known routes with real per-route metrics, not just the affected ones", async () => { + const result = await handleObserveImpact( + { predicted_severity: "degraded", affected_routes: ["/health"], window_seconds: 60 }, + makeFetcher({ + "/products": { errorRatePercent: 12, avgLatencyMs: 40, requestCount: 11 }, + "/orders": { errorRatePercent: 34, avgLatencyMs: 60, requestCount: 22 }, + "/health": { errorRatePercent: 0, avgLatencyMs: 3, requestCount: 33 }, + "/metrics": { errorRatePercent: 56, avgLatencyMs: 80, requestCount: 44 }, + }), + ); + assert.deepEqual(Object.keys(result.routes).sort(), [...KNOWN_ROUTES].sort()); + // Each route's metrics are the distinct, real values from the fetcher — + // not a shared placeholder — proving every route was actually queried. + assert.equal(result.routes["/products"].errorRatePercent, 12); + assert.equal(result.routes["/orders"].errorRatePercent, 34); + assert.equal(result.routes["/health"].errorRatePercent, 0); + assert.equal(result.routes["/metrics"].errorRatePercent, 56); +}); + +test("handleObserveImpact reports worse_than_predicted when observed severity exceeds the prediction", async () => { + const result = await handleObserveImpact( + { predicted_severity: "degraded", affected_routes: ["/products"], window_seconds: 60 }, + makeFetcher({ + "/products": { errorRatePercent: 92, avgLatencyMs: 200, requestCount: 30 }, + }), + ); + assert.equal(result.observedSeverity, "hard"); + assert.equal(result.verdict, "worse_than_predicted"); +}); + +test("registerTools surfaces a fetcher error as isError: true through the real McpServer/Client round trip", async () => { + const server = new McpServer({ name: "test-server-error", version: "1.0.0" }); + registerTools(server, async () => { + throw new Error("prometheus unreachable"); + }); + + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "test-client-error", version: "1.0.0" }); + await Promise.all([client.connect(clientTransport), server.connect(serverTransport)]); + + const result = await client.callTool({ + name: "observe_impact", + arguments: { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 60 }, + }); + assert.equal(result.isError, true); + + await client.close(); + await server.close(); +}); + test("registerTools registers observe_impact on a real McpServer and validates input via zod", async () => { const server = new McpServer({ name: "test-server", version: "1.0.0" }); registerTools(server, async () => ({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 })); From c7d90397182595332f8aa079a77ad84284ce6c96 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 10:57:03 +0530 Subject: [PATCH 16/21] docs: correct M5 status caveat, network-isolation claim, and opening tagline (I6, I7, M5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - M5's status bullet now discloses the same class of gap M3's already does: unit suite passes and the service typechecks cleanly, but the Docker image build, compose network wiring, and verify-m5.sh's live run are unverified against a real Docker daemon (Docker was unavailable throughout this milestone's implementation environment) (I6). - The network-isolation paragraph, written when only blast-radius-sandbox was isolated, is now factually wrong now that metrics-watcher is also off the default network and prometheus is dual-homed onto observability-net. Extended it to describe metrics-watcher's identical isolation and replaced "every other service shares" with an accurate three-way breakdown (mcp-server: none, needs full default access; blast-radius-sandbox and metrics-watcher: each on their own dedicated network) (I7). - Reworded the opening tagline's "observes impact in real time" — M5 explicitly rules out live/streaming observation as a non-goal; it samples Prometheus once, after the fact — to "checks real impact against its own prediction after the fact" (M5). --- README.md | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 106802b..0d6f333 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # Chaos Notary A chaos-engineering agent that runs destructive resilience experiments -against a live system, predicts blast radius before acting, observes impact -in real time, and stops for human approval before any irreversible action. +against a live system, predicts blast radius before acting, checks real +impact against its own prediction after the fact, and stops for human +approval before any irreversible action. The agent runs on [TrueForge](https://truefoundry.com) (an existing agent harness). This repo builds everything around it: the target stack to break, @@ -15,7 +16,7 @@ sandbox for blast-radius computation. - **M2 — MCP server**: done. Exposes chaos tools (container pause/stop/kill, network latency/loss injection) over Streamable HTTP, wrapping Pumba, with bounded duration per fault and guaranteed auto-revert. - **M3 — Agent + approval gates**: done. TrueForge agent manifest declares human approval for every destructive chaos tool; whether the gate actually fires is confirmed by manually running it against a live TrueForge instance, a check not yet performed in this repo — see [`agent/README.md`](agent/README.md) for setup and that verification walkthrough. - **M4 — Blast-radius sandbox**: done. Sealed read-only service for predicting chaos experiment blast radius, wired as a second MCP connector in the agent manifest, with zero Docker access to ensure it cannot affect the live stack. -- **M5 — Metrics-watcher**: done. Reports what Prometheus actually recorded after a chaos fault reverts and compares it against M4's prediction, wired as a third MCP connector in the agent manifest, closing the predict-then-observe loop. +- **M5 — Metrics-watcher**: done. Reports what Prometheus actually recorded after a chaos fault reverts and compares it against M4's prediction, wired as a third MCP connector in the agent manifest, closing the predict-then-observe loop. The unit test suite passes and the service typechecks cleanly, but the Docker image build, the docker-compose network wiring, and `scripts/verify-m5.sh`'s live run are unverified against a real Docker daemon — a check not yet performed in this repo (Docker was unavailable throughout this milestone's implementation environment). M6 (hardening) is not yet built. @@ -123,14 +124,25 @@ self-contained non-production demo stack, not an oversight. The `/mcp` endpoint itself has no authentication — any client that can reach it can pause, stop, or kill the allowlisted containers — so the published port is bound to `127.0.0.1` only (see `docker-compose.yml`), not exposed to the -wider network. `blast-radius-sandbox` is placed on its own dedicated -Compose network (`sandbox-net`), separate from the implicit default network -every other service shares — it makes zero outbound calls and depends on -nothing, so this costs it no reachability it actually needs, while ensuring -it cannot reach `mcp-server` (or anything else) by Docker's internal -service-name DNS even if a compromised or buggy dependency inside it tried -to. That keeps the "sealed sandbox" claim true at the network layer, not -just at the application layer (no Docker socket, read-only computation). +wider network. Three services now have deliberately restricted network +membership, each isolated on its own dedicated Compose network rather than +the implicit default network: `mcp-server` itself has none of these +restrictions — it needs full default-network access to reach the Docker +socket and the containers it manages — but `blast-radius-sandbox` is placed +on its own network (`sandbox-net`), and `metrics-watcher` is placed on its +own network (`observability-net`, shared only with `prometheus`, which is +dual-homed onto both `default` and `observability-net` so it can still be +scraped from and reached by `metrics-watcher`). `blast-radius-sandbox` +makes zero outbound calls and depends on nothing, so `sandbox-net` +isolation costs it no reachability it actually needs; `metrics-watcher`'s +only legitimate outbound call is to `prometheus`, so `observability-net` +isolation costs it nothing either. Both arrangements ensure neither service +can reach `mcp-server` (or anything else outside its own dedicated network) +by Docker's internal service-name DNS even if a compromised or buggy +dependency inside either tried to. That keeps the "sealed sandbox" claim +true at the network layer for `blast-radius-sandbox`, and the equivalent +claim true for `metrics-watcher`, not just at the application layer (no +Docker socket, read-only computation). ## M2 — MCP server tool surface From 90e5f6b1cb289cdaf2986ca59814610de6ffb13c Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 11:47:00 +0530 Subject: [PATCH 17/21] fix: ground severity classification in latency, api-health, and time-anchored queries (Qodo PR#5 findings 1-3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit classifyRoute only ever examined errorRatePercent, so a latency-only fault below checkout-api's 2000ms timeout produced no 5xx errors and reported observedSeverity "none" even when M4 predicted "degraded" (finding 1). Add a grounded OBSERVED_HARD_LATENCY_MS=2000 escalation (matches checkout-api's own connectionTimeoutMillis/query_timeout and M4's LATENCY_HARD_THRESHOLD_MS) so latency at/above that threshold now forces "hard" regardless of error rate. Deliberately no sub-2000ms "degraded latency" threshold is added — there's no grounded constant for that in this codebase. queryRouteMetrics built PromQL windows ending at "now" with no time anchor; a prior fix added a fixed +60s buffer, which mathematically dilutes short faults below the hard-error-rate threshold (a 30s fault in a 90s window is only 33% fault-period) — the one prior regression test used a 90s fault, masking this for shorter, equally valid experiments (finding 2). Add an optional offsetSeconds parameter to queryRouteMetrics (and the shared query() helper) that appends a PromQL `offset Ns` modifier to every range-vector selector, letting a caller anchor the window to a real elapsed time instead of guessing with a buffer. checkout-api's only request counter increments in Fastify's onResponse hook, which cannot fire while the process is paused/stopped/killed, so Prometheus's scrape fails entirely during that window and queryRouteMetrics returns nulls that classifyRoute reads as "none" — inverted for the single most common chaos scenario (finding 3). Add checkoutApiWasDown(), which checks Prometheus's own up{job="checkout-api"} scrape-health signal, and thread a new apiWasDown parameter through classifyRoute: null error-rate data now escalates to "hard" when the API was genuinely unreachable, rather than being silently read as idle traffic. --- services/metrics-watcher/src/prometheus.ts | 36 ++++++++++++++++++++-- services/metrics-watcher/src/severity.ts | 16 ++++++++-- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/services/metrics-watcher/src/prometheus.ts b/services/metrics-watcher/src/prometheus.ts index 4eb18be..745f1d1 100644 --- a/services/metrics-watcher/src/prometheus.ts +++ b/services/metrics-watcher/src/prometheus.ts @@ -37,10 +37,18 @@ export async function queryRouteMetrics( route: string, windowSeconds: number, fetchImpl: typeof fetch = fetch, + offsetSeconds: number = 0, ): Promise { + // offsetSeconds anchors the query window to end offsetSeconds ago instead + // of "now" — used when the caller supplies fault_ended_at, so the window + // covers the fault's own active period precisely rather than guessing + // with a fixed buffer. See PromQL `offset` modifier docs: it applies to + // the whole preceding range-vector selector. + const offsetClause = offsetSeconds > 0 ? ` offset ${offsetSeconds}s` : ""; + const requestCountRaw = await query( baseUrl, - `sum(increase(http_requests_total{route="${route}"}[${windowSeconds}s]))`, + `sum(increase(http_requests_total{route="${route}"}[${windowSeconds}s]${offsetClause}))`, fetchImpl, ); const requestCount = requestCountRaw ?? 0; @@ -50,13 +58,13 @@ export async function queryRouteMetrics( const errorRatePercent = await query( baseUrl, - `100 * (sum(rate(http_requests_total{route="${route}",status=~"5.."}[${windowSeconds}s])) or vector(0)) / sum(rate(http_requests_total{route="${route}"}[${windowSeconds}s]))`, + `100 * (sum(rate(http_requests_total{route="${route}",status=~"5.."}[${windowSeconds}s]${offsetClause})) or vector(0)) / sum(rate(http_requests_total{route="${route}"}[${windowSeconds}s]${offsetClause}))`, fetchImpl, ); const avgLatencySeconds = await query( baseUrl, - `sum(rate(http_request_duration_seconds_sum{route="${route}"}[${windowSeconds}s])) / sum(rate(http_request_duration_seconds_count{route="${route}"}[${windowSeconds}s]))`, + `sum(rate(http_request_duration_seconds_sum{route="${route}"}[${windowSeconds}s]${offsetClause})) / sum(rate(http_request_duration_seconds_count{route="${route}"}[${windowSeconds}s]${offsetClause}))`, fetchImpl, ); @@ -66,3 +74,25 @@ export async function queryRouteMetrics( requestCount, }; } + +// Checks Prometheus's own up{job="checkout-api"} scrape-health signal over +// the same window/offset used for route metrics. checkout-api's only +// request counter increments in Fastify's onResponse hook, which cannot +// fire while the process itself is paused/stopped/killed — so during a +// full-outage fault, Prometheus's scrape of checkout-api fails entirely and +// queryRouteMetrics sees no data at all (nulls), which is indistinguishable +// from "genuinely idle route" without this separate signal. +export async function checkoutApiWasDown( + baseUrl: string, + windowSeconds: number, + offsetSeconds: number, + fetchImpl: typeof fetch = fetch, +): Promise { + const offsetClause = offsetSeconds > 0 ? ` offset ${offsetSeconds}s` : ""; + const minUp = await query( + baseUrl, + `min_over_time(up{job="checkout-api"}[${windowSeconds}s]${offsetClause})`, + fetchImpl, + ); + return minUp !== null && minUp < 1; +} diff --git a/services/metrics-watcher/src/severity.ts b/services/metrics-watcher/src/severity.ts index 40b7da8..67f1838 100644 --- a/services/metrics-watcher/src/severity.ts +++ b/services/metrics-watcher/src/severity.ts @@ -2,13 +2,25 @@ import type { RouteMetrics } from "./prometheus.js"; export const OBSERVED_HARD_ERROR_RATE_PERCENT = 40; export const OBSERVED_NONE_ERROR_RATE_PERCENT = 1; +// Matches checkout-api's own connectionTimeoutMillis/query_timeout +// (services/checkout-api/src/server.ts) and M4's own LATENCY_HARD_THRESHOLD_MS +// (services/blast-radius-sandbox/src/topology.ts) — the same real physical +// constraint, not a new invented number. +export const OBSERVED_HARD_LATENCY_MS = 2000; export type ObservedSeverity = "hard" | "degraded" | "none"; export type PredictedSeverity = "hard" | "degraded"; export type Verdict = "matched" | "milder_than_predicted" | "worse_than_predicted"; -export function classifyRoute(metrics: RouteMetrics): ObservedSeverity { - if (metrics.errorRatePercent === null) return "none"; +// apiWasDown: true when Prometheus's own up{job="checkout-api"} scrape-health +// signal shows the target was unreachable for some part of the query window +// (see checkoutApiWasDown in prometheus.ts). A genuinely unreachable API is +// unambiguously worse than "no data" — it means the route couldn't even be +// scraped, not that it was idle — so null error-rate data escalates to +// "hard" instead of the default "none" in that case. +export function classifyRoute(metrics: RouteMetrics, apiWasDown: boolean = false): ObservedSeverity { + if (metrics.avgLatencyMs !== null && metrics.avgLatencyMs >= OBSERVED_HARD_LATENCY_MS) return "hard"; + if (metrics.errorRatePercent === null) return apiWasDown ? "hard" : "none"; if (metrics.errorRatePercent >= OBSERVED_HARD_ERROR_RATE_PERCENT) return "hard"; if (metrics.errorRatePercent <= OBSERVED_NONE_ERROR_RATE_PERCENT) return "none"; return "degraded"; From 14a8b4f5c8e83dc400d1127af8b6e8a14a82af61 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 11:47:08 +0530 Subject: [PATCH 18/21] fix: wire fault_ended_at time-anchoring and apiWasDown through observe_impact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an optional fault_ended_at (ISO-8601) input to observe_impact. When provided (the fault tool's own expiresAt, passed verbatim), the server computes offsetSeconds = max(0, round((Date.now() - Date.parse(fault_ended_at)) / 1000)) using its own clock and applies it to every route-metrics query and a single per-call checkoutApiWasDown() check, whose result now feeds every classifyRoute() call in the affected-routes computation. This lets window_seconds mean exactly the fault's own duration_seconds — no more manual buffer guessing, and it works correctly for short faults, not just long ones. Update the tool description to document fault_ended_at and the new hard-latency/api-outage escalations, keeping the old buffer guidance only as a fallback for callers with no timestamp to anchor with. Update the chaos-notary agent manifest's mandatory observe_impact workflow step to capture the fault tool's expiresAt and pass it as fault_ended_at with window_seconds set to exactly duration_seconds, removing the now- obsolete "+60s buffer" instruction (kept only as an explicit fallback). Update agent/README.md's manual verification walkthrough to match. --- agent/README.md | 34 ++++++++++--------- agent/chaos-notary.json | 2 +- services/metrics-watcher/src/tools.ts | 47 ++++++++++++++++++++++----- 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/agent/README.md b/agent/README.md index aaef7ac..b56f684 100644 --- a/agent/README.md +++ b/agent/README.md @@ -163,22 +163,24 @@ a real TrueForge instance before considering M3 done: known route `/products`, so `observe_impact` applies here and the agent should call it once Docker/fault state is confirmed clean in step 7: passing the `severity` from its earlier `predict_blast_radius` call as - `predicted_severity`, `["/products"]` as `affected_routes`, and - `window_seconds` set to the fault's `duration_seconds` **plus roughly 60 - seconds of buffer** (e.g. for this 30-second pause, expect something - around 90, not exactly 30 — a window that ends exactly at - `duration_seconds` risks landing entirely in the already-recovered - period once the seconds of reasoning/tool-call time between the fault - reverting and this call are accounted for). Separately, call - `observe_impact` yourself with those same arguments (predicted_severity - `"hard"`, affected_routes `["/products"]`, and a similarly buffered - window_seconds) and compare: the agent's closing report should state the - same real `observedSeverity` and `verdict` your direct call returns, not - just a repeat of what `predict_blast_radius` predicted earlier. If the - agent skips calling `observe_impact` when it does apply, passes a - `window_seconds` with no buffer, or its closing report only restates the - prediction instead of a real observed outcome, the wiring from Task 9 of - the M5 plan (and the C1/C2 fixes on top of it) isn't working. + `predicted_severity`, `["/products"]` as `affected_routes`, the + `expiresAt` value captured from the earlier `pause_container` response as + `fault_ended_at`, and `window_seconds` set to **exactly** the fault's + `duration_seconds` (e.g. `30` for this 30-second pause — no buffer + needed, since `fault_ended_at` lets the tool anchor its query window to + the fault's own active period precisely, using its own server-side + clock, regardless of how much reasoning/tool-call time has passed since + the fault reverted). Separately, call `observe_impact` yourself with + those same arguments (predicted_severity `"hard"`, affected_routes + `["/products"]`, the same `fault_ended_at`, and `window_seconds: 30`) and + compare: the agent's closing report should state the same real + `observedSeverity` and `verdict` your direct call returns, not just a + repeat of what `predict_blast_radius` predicted earlier. If the agent + skips calling `observe_impact` when it does apply, fails to pass + `fault_ended_at` when it captured a real `expiresAt`, or its closing + report only restates the prediction instead of a real observed outcome, + the wiring from Task 9 of the M5 plan (and the fix waves on top of it) + isn't working. Not every fault reaches this branch. If you instead run an experiment whose `predict_blast_radius affected` list contains no route-shaped diff --git a/agent/chaos-notary.json b/agent/chaos-notary.json index 56797cd..da569be 100644 --- a/agent/chaos-notary.json +++ b/agent/chaos-notary.json @@ -9,7 +9,7 @@ "temperature": 0.3 } }, - "instructions": "You are the Chaos Notary agent: you run bounded, reversible resilience\nexperiments against a live demo stack (the \"Chaos Notary target stack\") to\nreveal how it fails, and you always stop for explicit human approval\nbefore taking any action that could disrupt the system.\n\n## What you can do\n\nYou have access to 7 tools on the `mcp-server` connector, restricted to a\nfixed 5-container allowlist (chaos-pg-primary, chaos-pg-replica,\nchaos-checkout-api, chaos-prometheus, chaos-grafana):\n\n- `list_targets` — read-only. Reports each allowlisted container's current\n Docker state and whether a fault is currently active on it — not\n application health, only Docker/fault state. Always safe to call. Call\n it first before proposing any experiment, and again after one.\n- `pause_container`, `stop_container`, `kill_container` — freeze, stop, or\n kill a container's process for a bounded duration_seconds (5-300s),\n after which it automatically recovers on its own even if nothing calls\n you back.\n- `inject_latency`, `inject_packet_loss` — add network latency or packet\n loss to a container's traffic for the same bounded duration.\n- `clear_fault` — manually and immediately revert whatever fault is\n currently active on a container. Safe to call any time; it only ever\n restores safety, so it does not require approval.\n\nEvery fault-injection tool above (pause_container, stop_container,\nkill_container, inject_latency, inject_packet_loss) is destructive and\nrequires human approval before you're allowed to invoke it — you will be\nprompted automatically. Do not try to work around this or suggest the\nhuman disable it.\n\nYou also have access to one tool on the `blast-radius-sandbox` connector,\nread-only and side-effect-free — it never requires approval:\n\n- `predict_blast_radius(container, fault_kind, latency_ms?, jitter_ms?, percent?)` —\n given a container and a fault type, returns a static prediction:\n `severity` (\"hard\" or \"degraded\"), the list of `affected` targets and\n how each is affected, and the list of `unaffected` targets. It does not\n touch Docker or the target stack in any way — it's a lookup, not an\n experiment. Call it to ground your stated intent in this actual\n prediction, not your own general reasoning.\n\nYou also have access to one tool on the `metrics-watcher` connector, also\nread-only and side-effect-free — it never requires approval:\n\n- `observe_impact(predicted_severity, affected_routes, window_seconds)` —\n takes the `predicted_severity`, `affected_routes`, and `window_seconds`\n you give it and queries Prometheus for chaos-checkout-api's real error\n rate and latency over that window, returning `observedSeverity` and a\n `verdict` comparing what was actually observed against what was\n predicted. It reads Prometheus only — it does not touch Docker or the\n target stack in any way. `observedSeverity` and `verdict` are derived\n from error rate only — for a latency-only fault with no accompanying\n error spike, consult the response's per-route `avgLatencyMs` directly\n rather than relying on the verdict alone.\n\n## How to run an experiment\n\n1. Call `list_targets` first to confirm the target isn't already faulted.\n2. Call `predict_blast_radius` for the container and fault type you're\n considering. If the fault is `inject_latency` or `inject_packet_loss`,\n include `latency_ms`/`percent` matching the value you're about to\n propose — omitting them makes the prediction default to a milder\n 'degraded' severity that may not reflect what you're actually\n planning to run. If the proposed `inject_latency` experiment also\n uses jitter, include `jitter_ms` too — real jitter can push actual\n delays well past the hard-severity threshold even when the base\n `latency_ms` alone would not. Use its actual `severity`, `affected`, and\n `unaffected` fields — not your own general reasoning — to determine\n what to say next.\n3. State what you're about to do and why, in plain language, before\n proposing the tool call: which container, which fault type, how long,\n and what you expect to happen, based on that prediction (e.g.\n \"pausing chaos-pg-replica for 60s should make GET /products start\n returning 503s — predict_blast_radius reports it affected with\n severity 'hard', while POST /orders is unaffected\").\n4. Propose the fault-injection tool call. Wait for approval.\n5. After the call returns, tell the human where to watch for the effect:\n the Grafana dashboard at http://localhost:3001/d/chaos-notary\n (provisioned by the M1 target stack) shows the live error-rate panel.\n Once the fault has fully reverted, you will check the real impact\n yourself via `observe_impact` (step 7 below) — don't claim to have\n observed impact before then.\n6. Once the fault's duration_seconds has elapsed, call `list_targets`\n again to confirm the container's Docker status is running with no\n active fault. list_targets has no application-health signal — for\n confirmation the workload itself recovered, point the human at the\n dashboard. If Docker/fault state isn't clean, say so plainly and call\n `clear_fault` — that call never needs approval.\n7. Once Docker/fault state is confirmed clean, check whether `observe_impact`\n applies to this fault. Take `predict_blast_radius`'s `affected` targets\n and filter them down to only the entries that are actual checkout-api\n route paths: strip any leading HTTP verb from each (e.g. `\"GET /products\"`\n becomes `\"/products\"`), then keep only the ones that then exactly match\n one of the 4 known routes (`/products`, `/orders`, `/health`,\n `/metrics`). Some `affected` targets are NOT routes at all — e.g.\n `\"chaos-prometheus's scrape of /metrics\"`, `\"Grafana dashboard\n (chaos-notary)\"`, `\"Prometheus query API (:9090)\"`, or `\"Grafana\n dashboard UI\"` describe impact on Prometheus or Grafana themselves, not\n on a checkout-api route. Exclude these — never force-map a non-route\n target onto a route just because it looks similar.\n - If the filtered list has one or more routes in it, call\n `observe_impact`. This is mandatory whenever it applies — never skip\n it. Pass the `severity` from your earlier `predict_blast_radius` call\n as `predicted_severity`, the filtered route list as `affected_routes`,\n and `window_seconds` set to the fault's `duration_seconds` plus\n roughly 60 seconds of buffer (e.g. a 30-second fault should pass\n around 90, not exactly 30). The buffer matters: some amount of\n reasoning and tool-call time always passes between the fault reverting\n and this call being made, and you have no reliable way to measure that\n elapsed time precisely — a window that's exactly `duration_seconds`\n risks landing entirely in the already-recovered period and\n under-reporting severity, while a buffered window still reaches back\n far enough to cover the fault's actual active period. Report the\n tool's real `observedSeverity` and `verdict` to the human as the close\n of the experiment report — a real, checked outcome, not a restated\n prediction.\n - If the filtered list is empty — e.g. the fault's only predicted impact\n is on Grafana's dashboard, Prometheus's own query API, or it was a\n 0%-magnitude network fault with an empty `affected` list — do not call\n `observe_impact`. State plainly in your report that no route-level\n observation applies to this particular fault, since `observe_impact`\n only has visibility into chaos-checkout-api's own request-route\n metrics, not into Prometheus's or Grafana's own health.\n\n## What you must never do\n\n- Never call a fault-injection tool without first stating your intent and\n getting the approval prompt to fire.\n- Never target a container outside the fixed allowlist — the tools will\n reject it, but don't try anyway or suggest working around it.\n- Never claim you've verified an experiment's outcome by a means you don't\n actually have. `list_targets` only tells you Docker/fault state — never\n claim it told you a container is \"healthy\"; report exactly that, and\n point the human at the dashboard for anything beyond it. `observe_impact`\n is different: its `observedSeverity`, `verdict`, and per-route numbers\n come from a real, current Prometheus query, so you may cite them\n directly once you have actually called it — never state them before\n calling it, and never state anything beyond what its actual return\n value contains.\n", + "instructions": "You are the Chaos Notary agent: you run bounded, reversible resilience\nexperiments against a live demo stack (the \"Chaos Notary target stack\") to\nreveal how it fails, and you always stop for explicit human approval\nbefore taking any action that could disrupt the system.\n\n## What you can do\n\nYou have access to 7 tools on the `mcp-server` connector, restricted to a\nfixed 5-container allowlist (chaos-pg-primary, chaos-pg-replica,\nchaos-checkout-api, chaos-prometheus, chaos-grafana):\n\n- `list_targets` — read-only. Reports each allowlisted container's current\n Docker state and whether a fault is currently active on it — not\n application health, only Docker/fault state. Always safe to call. Call\n it first before proposing any experiment, and again after one.\n- `pause_container`, `stop_container`, `kill_container` — freeze, stop, or\n kill a container's process for a bounded duration_seconds (5-300s),\n after which it automatically recovers on its own even if nothing calls\n you back.\n- `inject_latency`, `inject_packet_loss` — add network latency or packet\n loss to a container's traffic for the same bounded duration.\n- `clear_fault` — manually and immediately revert whatever fault is\n currently active on a container. Safe to call any time; it only ever\n restores safety, so it does not require approval.\n\nEvery fault-injection tool above (pause_container, stop_container,\nkill_container, inject_latency, inject_packet_loss) is destructive and\nrequires human approval before you're allowed to invoke it — you will be\nprompted automatically. Do not try to work around this or suggest the\nhuman disable it.\n\nYou also have access to one tool on the `blast-radius-sandbox` connector,\nread-only and side-effect-free — it never requires approval:\n\n- `predict_blast_radius(container, fault_kind, latency_ms?, jitter_ms?, percent?)` —\n given a container and a fault type, returns a static prediction:\n `severity` (\"hard\" or \"degraded\"), the list of `affected` targets and\n how each is affected, and the list of `unaffected` targets. It does not\n touch Docker or the target stack in any way — it's a lookup, not an\n experiment. Call it to ground your stated intent in this actual\n prediction, not your own general reasoning.\n\nYou also have access to one tool on the `metrics-watcher` connector, also\nread-only and side-effect-free — it never requires approval:\n\n- `observe_impact(predicted_severity, affected_routes, window_seconds, fault_ended_at?)` —\n takes the `predicted_severity`, `affected_routes`, and `window_seconds`\n you give it and queries Prometheus for chaos-checkout-api's real error\n rate and latency over that window, returning `observedSeverity` and a\n `verdict` comparing what was actually observed against what was\n predicted. It reads Prometheus only — it does not touch Docker or the\n target stack in any way. Pass `fault_ended_at` as the fault tool's own\n `expiresAt` timestamp (verbatim) to precisely time-anchor the query\n window to the fault's actual active period; with it, `window_seconds`\n should be exactly the fault's `duration_seconds` (no buffer needed).\n `observedSeverity`/`verdict` escalate to \"hard\" when `avgLatencyMs`\n reaches 2000ms regardless of error rate, and also when checkout-api's\n own Prometheus scrape was down during the window (a real outage, not\n idle traffic) — but a latency-only fault that stays below 2000ms with\n no accompanying error spike is still only visible via the response's\n per-route `avgLatencyMs` directly, not via the verdict alone.\n\n## How to run an experiment\n\n1. Call `list_targets` first to confirm the target isn't already faulted.\n2. Call `predict_blast_radius` for the container and fault type you're\n considering. If the fault is `inject_latency` or `inject_packet_loss`,\n include `latency_ms`/`percent` matching the value you're about to\n propose — omitting them makes the prediction default to a milder\n 'degraded' severity that may not reflect what you're actually\n planning to run. If the proposed `inject_latency` experiment also\n uses jitter, include `jitter_ms` too — real jitter can push actual\n delays well past the hard-severity threshold even when the base\n `latency_ms` alone would not. Use its actual `severity`, `affected`, and\n `unaffected` fields — not your own general reasoning — to determine\n what to say next.\n3. State what you're about to do and why, in plain language, before\n proposing the tool call: which container, which fault type, how long,\n and what you expect to happen, based on that prediction (e.g.\n \"pausing chaos-pg-replica for 60s should make GET /products start\n returning 503s — predict_blast_radius reports it affected with\n severity 'hard', while POST /orders is unaffected\").\n4. Propose the fault-injection tool call. Wait for approval. Once it\n returns, capture the exact `expiresAt` value from its response — you\n will need to pass it verbatim as `fault_ended_at` in step 7.\n5. After the call returns, tell the human where to watch for the effect:\n the Grafana dashboard at http://localhost:3001/d/chaos-notary\n (provisioned by the M1 target stack) shows the live error-rate panel.\n Once the fault has fully reverted, you will check the real impact\n yourself via `observe_impact` (step 7 below) — don't claim to have\n observed impact before then.\n6. Once the fault's duration_seconds has elapsed, call `list_targets`\n again to confirm the container's Docker status is running with no\n active fault. list_targets has no application-health signal — for\n confirmation the workload itself recovered, point the human at the\n dashboard. If Docker/fault state isn't clean, say so plainly and call\n `clear_fault` — that call never needs approval.\n7. Once Docker/fault state is confirmed clean, check whether `observe_impact`\n applies to this fault. Take `predict_blast_radius`'s `affected` targets\n and filter them down to only the entries that are actual checkout-api\n route paths: strip any leading HTTP verb from each (e.g. `\"GET /products\"`\n becomes `\"/products\"`), then keep only the ones that then exactly match\n one of the 4 known routes (`/products`, `/orders`, `/health`,\n `/metrics`). Some `affected` targets are NOT routes at all — e.g.\n `\"chaos-prometheus's scrape of /metrics\"`, `\"Grafana dashboard\n (chaos-notary)\"`, `\"Prometheus query API (:9090)\"`, or `\"Grafana\n dashboard UI\"` describe impact on Prometheus or Grafana themselves, not\n on a checkout-api route. Exclude these — never force-map a non-route\n target onto a route just because it looks similar.\n - If the filtered list has one or more routes in it, call\n `observe_impact`. This is mandatory whenever it applies — never skip\n it. Pass the `severity` from your earlier `predict_blast_radius` call\n as `predicted_severity`, the filtered route list as `affected_routes`,\n the `expiresAt` value you captured in step 4 as `fault_ended_at`, and\n `window_seconds` set to exactly the fault's `duration_seconds` (no\n buffer needed). Passing `fault_ended_at` lets the tool precisely\n anchor its Prometheus query window to the fault's own active period\n using its own server-side clock, regardless of how much reasoning or\n tool-call time has passed since the fault reverted — you don't need to\n estimate elapsed time yourself. If for some reason you don't have the\n `expiresAt` value (e.g. it was missing from the tool response), omit\n `fault_ended_at` and fall back to passing `window_seconds` as\n `duration_seconds` plus roughly 60 seconds of buffer instead, since\n without a timestamp to anchor with, a window that's exactly\n `duration_seconds` risks landing entirely in the already-recovered\n period. Report the tool's real `observedSeverity` and `verdict` to the\n human as the close of the experiment report — a real, checked outcome,\n not a restated prediction.\n - If the filtered list is empty — e.g. the fault's only predicted impact\n is on Grafana's dashboard, Prometheus's own query API, or it was a\n 0%-magnitude network fault with an empty `affected` list — do not call\n `observe_impact`. State plainly in your report that no route-level\n observation applies to this particular fault, since `observe_impact`\n only has visibility into chaos-checkout-api's own request-route\n metrics, not into Prometheus's or Grafana's own health.\n\n## What you must never do\n\n- Never call a fault-injection tool without first stating your intent and\n getting the approval prompt to fire.\n- Never target a container outside the fixed allowlist — the tools will\n reject it, but don't try anyway or suggest working around it.\n- Never claim you've verified an experiment's outcome by a means you don't\n actually have. `list_targets` only tells you Docker/fault state — never\n claim it told you a container is \"healthy\"; report exactly that, and\n point the human at the dashboard for anything beyond it. `observe_impact`\n is different: its `observedSeverity`, `verdict`, and per-route numbers\n come from a real, current Prometheus query, so you may cite them\n directly once you have actually called it — never state them before\n calling it, and never state anything beyond what its actual return\n value contains.\n", "mcp_servers": [ { "type": "truefoundry-mcp-registry", diff --git a/services/metrics-watcher/src/tools.ts b/services/metrics-watcher/src/tools.ts index 25ce7e4..54d1b6e 100644 --- a/services/metrics-watcher/src/tools.ts +++ b/services/metrics-watcher/src/tools.ts @@ -1,7 +1,7 @@ // services/metrics-watcher/src/tools.ts import { z } from "zod"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; -import { queryRouteMetrics, type RouteMetrics } from "./prometheus.js"; +import { queryRouteMetrics, checkoutApiWasDown, type RouteMetrics } from "./prometheus.js"; import { classifyRoute, worstOf, computeVerdict, type ObservedSeverity, type Verdict } from "./severity.js"; export const KNOWN_ROUTES = ["/products", "/orders", "/health", "/metrics"] as const; @@ -13,6 +13,7 @@ export interface ObserveImpactArgs { predicted_severity: "hard" | "degraded"; affected_routes: KnownRoute[]; window_seconds: number; + fault_ended_at?: string; } export interface ObserveImpactResult { @@ -23,21 +24,45 @@ export interface ObserveImpactResult { routes: Record; } -type RouteMetricsFetcher = (route: KnownRoute) => Promise; +// Both fetchers take the per-call offsetSeconds explicitly (rather than +// having it baked silently into a closure) so callers — including tests — +// can observe exactly what offset a given fault_ended_at was translated +// into and that it was actually threaded through to every Prometheus query. +type RouteMetricsFetcher = (route: KnownRoute, offsetSeconds: number) => Promise; +type ApiHealthFetcher = (windowSeconds: number, offsetSeconds: number) => Promise; -// window_seconds varies per call, so the default fetcher is built per-call -// inside handleObserveImpact's default parameter (below), not as a single +// window_seconds varies per call, so the default fetchers are built per-call +// inside handleObserveImpact's default parameters (below), not as a single // top-level constant. export async function handleObserveImpact( args: ObserveImpactArgs, - fetchRouteMetrics: RouteMetricsFetcher = (route) => queryRouteMetrics(PROMETHEUS_URL, route, args.window_seconds), + fetchRouteMetrics: RouteMetricsFetcher = (route, offsetSeconds) => + queryRouteMetrics(PROMETHEUS_URL, route, args.window_seconds, undefined, offsetSeconds), + fetchApiHealth: ApiHealthFetcher = (windowSeconds, offsetSeconds) => + checkoutApiWasDown(PROMETHEUS_URL, windowSeconds, offsetSeconds), ): Promise { + // When fault_ended_at is supplied (the fault tool's own expiresAt, + // verbatim), anchor the query window to end there instead of "now" — the + // server computes the real elapsed offset with its own Date.now(), so the + // caller never has to estimate elapsed time itself. Without it, behavior + // is unchanged: window ends at "now" (offsetSeconds 0), still useful for + // mid-fault or ad-hoc calls. + const offsetSeconds = args.fault_ended_at + ? Math.max(0, Math.round((Date.now() - Date.parse(args.fault_ended_at)) / 1000)) + : 0; + const routes = {} as Record; for (const route of KNOWN_ROUTES) { - routes[route] = await fetchRouteMetrics(route); + routes[route] = await fetchRouteMetrics(route, offsetSeconds); } - const affectedSeverities = args.affected_routes.map((route) => classifyRoute(routes[route])); + // A single API-wide scrape-health check, not one per route: checkout-api's + // request counters can't increment while the process itself is + // paused/stopped/killed, so a route with null error-rate data during a + // full outage must be escalated to "hard" rather than read as "none". + const apiWasDown = await fetchApiHealth(args.window_seconds, offsetSeconds); + + const affectedSeverities = args.affected_routes.map((route) => classifyRoute(routes[route], apiWasDown)); const observedSeverity = worstOf(affectedSeverities); const verdict = computeVerdict(args.predicted_severity, observedSeverity); @@ -53,21 +78,25 @@ export async function handleObserveImpact( export function registerTools( server: McpServer, fetchRouteMetrics?: RouteMetricsFetcher, + fetchApiHealth?: ApiHealthFetcher, ): void { server.registerTool( "observe_impact", { title: "Observe real chaos-experiment impact", description: - "Query Prometheus for checkout-api's real, currently-measured error rate and latency per route, and compare against a predicted severity from predict_blast_radius. Call this after a fault reverts, passing the same predicted_severity and affected routes you got from predict_blast_radius (stripped of any leading HTTP verb, e.g. 'GET /products' becomes '/products'; only pass targets that are actual routes — some predict_blast_radius targets describe impact on Prometheus/Grafana themselves and are not routes at all, so skip calling this tool if none of the affected targets are routes). Pass window_seconds with some margin beyond the fault's own duration_seconds (e.g. duration_seconds + ~60s) to account for the delay between the fault ending and this call being made — a window equal to exactly duration_seconds risks landing entirely in the already-recovered period. Note observedSeverity/verdict reflect error rate only; for a latency-only fault with no accompanying error spike, consult the response's per-route avgLatencyMs directly rather than relying on the verdict alone.", + "Query Prometheus for checkout-api's real, currently-measured error rate and latency per route, and compare against a predicted severity from predict_blast_radius. Call this after a fault reverts, passing the same predicted_severity and affected routes you got from predict_blast_radius (stripped of any leading HTTP verb, e.g. 'GET /products' becomes '/products'; only pass targets that are actual routes — some predict_blast_radius targets describe impact on Prometheus/Grafana themselves and are not routes at all, so skip calling this tool if none of the affected targets are routes). Pass fault_ended_at as the exact expiresAt timestamp (ISO-8601) returned by the fault-injection tool call, if you have it — this precisely time-anchors the query window to the fault's own active period regardless of how much reasoning/tool-call time has since passed, and with it window_seconds should be set to exactly the fault's duration_seconds (no buffer needed). If you don't have an expiresAt to anchor with, omit fault_ended_at and instead pass window_seconds with some margin beyond duration_seconds (e.g. duration_seconds + ~60s) as a fallback, since the window will then end at 'now' and a window equal to exactly duration_seconds risks landing entirely in the already-recovered period. Note observedSeverity/verdict now also escalate to 'hard' when avgLatencyMs reaches 2000ms (matching checkout-api's own timeout) regardless of error rate, and separately when checkout-api's own Prometheus scrape was down during the window (a full outage, not idle traffic) — but a latency-only fault that stays below 2000ms with no accompanying error spike is still only visible via the response's per-route avgLatencyMs directly, not via observedSeverity/verdict alone.", inputSchema: { predicted_severity: z.enum(["hard", "degraded"]), affected_routes: z.array(z.enum(KNOWN_ROUTES)).min(1), window_seconds: z.number().int().positive(), + fault_ended_at: z.string().datetime().optional(), }, }, async (args) => ({ - content: [{ type: "text", text: JSON.stringify(await handleObserveImpact(args, fetchRouteMetrics)) }], + content: [ + { type: "text", text: JSON.stringify(await handleObserveImpact(args, fetchRouteMetrics, fetchApiHealth)) }, + ], }), ); } From 0829bc55ff9fe9abf19da5315a2201c8374a5451 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 11:47:16 +0530 Subject: [PATCH 19/21] test: cover latency-hard-threshold, offset anchoring, and apiWasDown escalation severity.test.ts: avgLatencyMs at/above 2000ms classifies as hard regardless of error rate; just below 2000ms does not escalate via latency alone (exact boundary); null error rate + apiWasDown true classifies as hard; apiWasDown false/omitted preserves existing "none" behavior. prometheus.test.ts: queryRouteMetrics includes an `offset Ns` clause on every range-vector query when offsetSeconds > 0 and omits it when 0/omitted; checkoutApiWasDown returns true/false/false for up{} values of 0/1/no-data, and its own query carries the offset clause correctly. tools.test.ts: updated the fake route-metrics fetcher and every call site to the new (route, offsetSeconds) signature, added a fake api-health fetcher to every handleObserveImpact/registerTools call (the real default would otherwise hit the network in tests), and added cases proving fault_ended_at is parsed into the correct offsetSeconds and threaded through to every route fetch plus the single per-call api-health check, that a full outage (apiWasDown true) escalates null route metrics to hard, and that zod rejects a malformed fault_ended_at. 19 tests -> 33 tests, all passing. --- .../metrics-watcher/test/prometheus.test.ts | 95 +++++++++- .../metrics-watcher/test/severity.test.ts | 21 +++ services/metrics-watcher/test/tools.test.ts | 172 +++++++++++++++--- 3 files changed, 264 insertions(+), 24 deletions(-) diff --git a/services/metrics-watcher/test/prometheus.test.ts b/services/metrics-watcher/test/prometheus.test.ts index 57debff..73f7a45 100644 --- a/services/metrics-watcher/test/prometheus.test.ts +++ b/services/metrics-watcher/test/prometheus.test.ts @@ -1,7 +1,7 @@ // services/metrics-watcher/test/prometheus.test.ts import { test } from "node:test"; import assert from "node:assert/strict"; -import { queryRouteMetrics } from "../src/prometheus.js"; +import { queryRouteMetrics, checkoutApiWasDown } from "../src/prometheus.js"; function fakePrometheus(responses: Record) { return async (url: string | URL) => { @@ -26,6 +26,31 @@ function fakePrometheus(responses: Record) { }; } +// Captures every PromQL query string sent, rather than dispatching on it, so +// tests can assert on the exact query text (e.g. presence/absence of the +// `offset Ns` clause) instead of just the returned value. +function capturingFetcher(value: string | null) { + const queries: string[] = []; + const fetchImpl = async (url: string | URL) => { + const u = new URL(url); + queries.push(u.searchParams.get("query") ?? ""); + if (value === null) { + return new Response( + JSON.stringify({ status: "success", data: { resultType: "vector", result: [] } }), + { status: 200 }, + ); + } + return new Response( + JSON.stringify({ + status: "success", + data: { resultType: "vector", result: [{ metric: {}, value: [1700000000, value] }] }, + }), + { status: 200 }, + ); + }; + return { fetchImpl, queries }; +} + test("queryRouteMetrics returns real numbers when Prometheus has data", async () => { const fetchImpl = fakePrometheus({ "increase(http_requests_total": "12", @@ -62,3 +87,71 @@ test("queryRouteMetrics throws when Prometheus responds with a non-success statu /Prometheus query returned status/, ); }); + +test("queryRouteMetrics includes an `offset Ns` clause on every range-vector query when offsetSeconds > 0", async () => { + const { fetchImpl, queries } = capturingFetcher("5"); + await queryRouteMetrics("http://prometheus:9090", "/products", 60, fetchImpl as unknown as typeof fetch, 45); + assert.equal(queries.length, 3, "expects requestCount, errorRate, and latency queries"); + for (const q of queries) { + assert.match(q, /\[60s\] offset 45s/, `query missing offset clause: ${q}`); + } +}); + +test("queryRouteMetrics omits the offset clause when offsetSeconds is 0 or omitted", async () => { + const { fetchImpl: fetchImplDefault, queries: queriesDefault } = capturingFetcher("5"); + await queryRouteMetrics("http://prometheus:9090", "/products", 60, fetchImplDefault as unknown as typeof fetch); + assert.ok(queriesDefault.length > 0); + for (const q of queriesDefault) { + assert.ok(!q.includes("offset"), `query unexpectedly contains offset clause: ${q}`); + } + + const { fetchImpl: fetchImplZero, queries: queriesZero } = capturingFetcher("5"); + await queryRouteMetrics("http://prometheus:9090", "/products", 60, fetchImplZero as unknown as typeof fetch, 0); + for (const q of queriesZero) { + assert.ok(!q.includes("offset"), `query unexpectedly contains offset clause: ${q}`); + } +}); + +test("checkoutApiWasDown returns true when up{} reports below 1 (scrape target down) during the window", async () => { + const fetchImpl = fakePrometheus({ 'up{job="checkout-api"}': "0" }); + const result = await checkoutApiWasDown( + "http://prometheus:9090", + 60, + 30, + fetchImpl as unknown as typeof fetch, + ); + assert.equal(result, true); +}); + +test("checkoutApiWasDown returns false when up{} reports 1 (scrape target healthy) throughout the window", async () => { + const fetchImpl = fakePrometheus({ 'up{job="checkout-api"}': "1" }); + const result = await checkoutApiWasDown( + "http://prometheus:9090", + 60, + 30, + fetchImpl as unknown as typeof fetch, + ); + assert.equal(result, false); +}); + +test("checkoutApiWasDown returns false when there is no up{} data at all (treated as not-known-down)", async () => { + const fetchImpl = fakePrometheus({}); + const result = await checkoutApiWasDown( + "http://prometheus:9090", + 60, + 30, + fetchImpl as unknown as typeof fetch, + ); + assert.equal(result, false); +}); + +test("checkoutApiWasDown's query includes the offset clause when offsetSeconds > 0, and omits it when 0", async () => { + const { fetchImpl, queries } = capturingFetcher("0"); + await checkoutApiWasDown("http://prometheus:9090", 60, 20, fetchImpl as unknown as typeof fetch); + assert.equal(queries.length, 1); + assert.match(queries[0], /min_over_time\(up\{job="checkout-api"\}\[60s\] offset 20s\)/); + + const { fetchImpl: fetchImplNoOffset, queries: queriesNoOffset } = capturingFetcher("0"); + await checkoutApiWasDown("http://prometheus:9090", 60, 0, fetchImplNoOffset as unknown as typeof fetch); + assert.ok(!queriesNoOffset[0].includes("offset")); +}); diff --git a/services/metrics-watcher/test/severity.test.ts b/services/metrics-watcher/test/severity.test.ts index f6d44cb..619bc70 100644 --- a/services/metrics-watcher/test/severity.test.ts +++ b/services/metrics-watcher/test/severity.test.ts @@ -45,3 +45,24 @@ test("computeVerdict: observed is milder than predicted", () => { test("computeVerdict: observed is worse than predicted", () => { assert.equal(computeVerdict("degraded", "hard"), "worse_than_predicted"); }); + +test("classifyRoute: avgLatencyMs at or above 2000ms classifies as hard even with zero/null error rate", () => { + assert.equal(classifyRoute({ errorRatePercent: 0, avgLatencyMs: 2000, requestCount: 5 }), "hard"); + assert.equal(classifyRoute({ errorRatePercent: null, avgLatencyMs: 2500, requestCount: 5 }), "hard"); +}); + +test("classifyRoute: avgLatencyMs just below 2000ms does not escalate via latency alone (boundary is exact)", () => { + // Still governed by error rate as before — a low error rate with + // sub-threshold latency must NOT classify as hard. + assert.equal(classifyRoute({ errorRatePercent: 0, avgLatencyMs: 1999, requestCount: 5 }), "none"); + assert.equal(classifyRoute({ errorRatePercent: 1.5, avgLatencyMs: 1999, requestCount: 5 }), "degraded"); +}); + +test("classifyRoute: null error rate with apiWasDown true classifies as hard (genuine outage, not idle)", () => { + assert.equal(classifyRoute({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, true), "hard"); +}); + +test("classifyRoute: null error rate with apiWasDown false (or omitted) keeps existing none behavior", () => { + assert.equal(classifyRoute({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, false), "none"); + assert.equal(classifyRoute({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }), "none"); +}); diff --git a/services/metrics-watcher/test/tools.test.ts b/services/metrics-watcher/test/tools.test.ts index 2cf1ead..2169322 100644 --- a/services/metrics-watcher/test/tools.test.ts +++ b/services/metrics-watcher/test/tools.test.ts @@ -11,18 +11,41 @@ import type { RouteMetrics } from "../src/prometheus.js"; // argument, this one dispatches on it, so tests can prove observe_impact's // severity computation is actually scoped to affected_routes and that the // `routes` field in the response reflects each route's own real metrics. +// It also records the offsetSeconds it was called with, so tests can assert +// fault_ended_at was correctly translated into an offset and threaded +// through to route-metrics fetching. const BASELINE: RouteMetrics = { errorRatePercent: 0, avgLatencyMs: 5, requestCount: 100 }; function makeFetcher(overrides: Partial>) { - return async (route: KnownRoute): Promise => overrides[route] ?? BASELINE; + const offsetsSeen: number[] = []; + const fetcher = async (route: KnownRoute, offsetSeconds: number): Promise => { + offsetsSeen.push(offsetSeconds); + return overrides[route] ?? BASELINE; + }; + return { fetcher, offsetsSeen }; +} + +// checkout-api's own scrape-health fetcher — most tests don't care about +// this signal, so a stub that always reports "not down" and records what +// window/offset it was called with is enough to isolate the behavior under +// test from the real network-backed default. +function makeApiHealthFetcher(apiWasDown = false) { + const callsSeen: Array<{ windowSeconds: number; offsetSeconds: number }> = []; + const fetcher = async (windowSeconds: number, offsetSeconds: number): Promise => { + callsSeen.push({ windowSeconds, offsetSeconds }); + return apiWasDown; + }; + return { fetcher, callsSeen }; } test("handleObserveImpact reports hard severity and a matched verdict when the affected route is failing hard", async () => { + const { fetcher } = makeFetcher({ + "/products": { errorRatePercent: 87, avgLatencyMs: 5, requestCount: 20 }, + }); const result = await handleObserveImpact( { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 60 }, - makeFetcher({ - "/products": { errorRatePercent: 87, avgLatencyMs: 5, requestCount: 20 }, - }), + fetcher, + makeApiHealthFetcher().fetcher, ); assert.equal(result.observedSeverity, "hard"); assert.equal(result.verdict, "matched"); @@ -30,14 +53,16 @@ test("handleObserveImpact reports hard severity and a matched verdict when the a }); test("handleObserveImpact scopes observedSeverity to affected_routes only, ignoring unrelated routes' severity", async () => { + const { fetcher } = makeFetcher({ + // /products is failing hard, but it's NOT in affected_routes. + "/products": { errorRatePercent: 90, avgLatencyMs: 5, requestCount: 50 }, + // /orders — the only affected route — has no errors at all. + "/orders": { errorRatePercent: 0, avgLatencyMs: 5, requestCount: 50 }, + }); const result = await handleObserveImpact( { predicted_severity: "hard", affected_routes: ["/orders"], window_seconds: 60 }, - makeFetcher({ - // /products is failing hard, but it's NOT in affected_routes. - "/products": { errorRatePercent: 90, avgLatencyMs: 5, requestCount: 50 }, - // /orders — the only affected route — has no errors at all. - "/orders": { errorRatePercent: 0, avgLatencyMs: 5, requestCount: 50 }, - }), + fetcher, + makeApiHealthFetcher().fetcher, ); assert.equal(result.observedSeverity, "none"); assert.equal(result.verdict, "milder_than_predicted"); @@ -47,14 +72,16 @@ test("handleObserveImpact scopes observedSeverity to affected_routes only, ignor }); test("handleObserveImpact's routes field always contains all 4 known routes with real per-route metrics, not just the affected ones", async () => { + const { fetcher } = makeFetcher({ + "/products": { errorRatePercent: 12, avgLatencyMs: 40, requestCount: 11 }, + "/orders": { errorRatePercent: 34, avgLatencyMs: 60, requestCount: 22 }, + "/health": { errorRatePercent: 0, avgLatencyMs: 3, requestCount: 33 }, + "/metrics": { errorRatePercent: 56, avgLatencyMs: 80, requestCount: 44 }, + }); const result = await handleObserveImpact( { predicted_severity: "degraded", affected_routes: ["/health"], window_seconds: 60 }, - makeFetcher({ - "/products": { errorRatePercent: 12, avgLatencyMs: 40, requestCount: 11 }, - "/orders": { errorRatePercent: 34, avgLatencyMs: 60, requestCount: 22 }, - "/health": { errorRatePercent: 0, avgLatencyMs: 3, requestCount: 33 }, - "/metrics": { errorRatePercent: 56, avgLatencyMs: 80, requestCount: 44 }, - }), + fetcher, + makeApiHealthFetcher().fetcher, ); assert.deepEqual(Object.keys(result.routes).sort(), [...KNOWN_ROUTES].sort()); // Each route's metrics are the distinct, real values from the fetcher — @@ -66,21 +93,94 @@ test("handleObserveImpact's routes field always contains all 4 known routes with }); test("handleObserveImpact reports worse_than_predicted when observed severity exceeds the prediction", async () => { + const { fetcher } = makeFetcher({ + "/products": { errorRatePercent: 92, avgLatencyMs: 200, requestCount: 30 }, + }); const result = await handleObserveImpact( { predicted_severity: "degraded", affected_routes: ["/products"], window_seconds: 60 }, - makeFetcher({ - "/products": { errorRatePercent: 92, avgLatencyMs: 200, requestCount: 30 }, - }), + fetcher, + makeApiHealthFetcher().fetcher, ); assert.equal(result.observedSeverity, "hard"); assert.equal(result.verdict, "worse_than_predicted"); }); +test("handleObserveImpact escalates to hard when checkoutApiWasDown is true, even with null route metrics", async () => { + // No fault_ended_at, so requestCount/errorRatePercent come back null via + // the BASELINE-overriding fetcher below (simulating checkout-api's own + // request counters never incrementing during a full outage). + const { fetcher } = makeFetcher({ + "/products": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, + }); + const result = await handleObserveImpact( + { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 60 }, + fetcher, + makeApiHealthFetcher(true).fetcher, + ); + assert.equal(result.observedSeverity, "hard"); + assert.equal(result.verdict, "matched"); +}); + +test("handleObserveImpact keeps none for null route metrics when checkoutApiWasDown is false (genuinely idle route)", async () => { + const { fetcher } = makeFetcher({ + "/products": { errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }, + }); + const result = await handleObserveImpact( + { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 60 }, + fetcher, + makeApiHealthFetcher(false).fetcher, + ); + assert.equal(result.observedSeverity, "none"); + assert.equal(result.verdict, "milder_than_predicted"); +}); + +test("handleObserveImpact computes offsetSeconds from fault_ended_at and passes it through to route-metrics/api-health fetching", async () => { + const { fetcher, offsetsSeen } = makeFetcher({}); + const { fetcher: apiHealthFetcher, callsSeen } = makeApiHealthFetcher(); + + const faultEndedAt = new Date(Date.now() - 45_000).toISOString(); // ended 45s ago + await handleObserveImpact( + { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 30, fault_ended_at: faultEndedAt }, + fetcher, + apiHealthFetcher, + ); + + // Every route fetch (all 4 known routes) should see the same computed offset. + assert.equal(offsetsSeen.length, KNOWN_ROUTES.length); + for (const offset of offsetsSeen) { + // Allow a small amount of slack for real elapsed test-execution time. + assert.ok(offset >= 44 && offset <= 47, `expected offset ~45, got ${offset}`); + } + assert.equal(callsSeen.length, 1, "api health should be checked once per call, not once per route"); + assert.equal(callsSeen[0].windowSeconds, 30); + assert.ok(callsSeen[0].offsetSeconds >= 44 && callsSeen[0].offsetSeconds <= 47); +}); + +test("handleObserveImpact uses offsetSeconds 0 when fault_ended_at is omitted", async () => { + const { fetcher, offsetsSeen } = makeFetcher({}); + const { fetcher: apiHealthFetcher, callsSeen } = makeApiHealthFetcher(); + + await handleObserveImpact( + { predicted_severity: "hard", affected_routes: ["/products"], window_seconds: 60 }, + fetcher, + apiHealthFetcher, + ); + + for (const offset of offsetsSeen) { + assert.equal(offset, 0); + } + assert.equal(callsSeen[0].offsetSeconds, 0); +}); + test("registerTools surfaces a fetcher error as isError: true through the real McpServer/Client round trip", async () => { const server = new McpServer({ name: "test-server-error", version: "1.0.0" }); - registerTools(server, async () => { - throw new Error("prometheus unreachable"); - }); + registerTools( + server, + async () => { + throw new Error("prometheus unreachable"); + }, + async () => false, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); const client = new Client({ name: "test-client-error", version: "1.0.0" }); @@ -98,7 +198,11 @@ test("registerTools surfaces a fetcher error as isError: true through the real M test("registerTools registers observe_impact on a real McpServer and validates input via zod", async () => { const server = new McpServer({ name: "test-server", version: "1.0.0" }); - registerTools(server, async () => ({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 })); + registerTools( + server, + async () => ({ errorRatePercent: null, avgLatencyMs: null, requestCount: 0 }), + async () => false, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); const client = new Client({ name: "test-client", version: "1.0.0" }); @@ -116,6 +220,28 @@ test("registerTools registers observe_impact on a real McpServer and validates i }); assert.equal(good.isError, undefined); + const goodWithFaultEndedAt = await client.callTool({ + name: "observe_impact", + arguments: { + predicted_severity: "degraded", + affected_routes: ["/products"], + window_seconds: 60, + fault_ended_at: new Date().toISOString(), + }, + }); + assert.equal(goodWithFaultEndedAt.isError, undefined); + + const badFaultEndedAt = await client.callTool({ + name: "observe_impact", + arguments: { + predicted_severity: "degraded", + affected_routes: ["/products"], + window_seconds: 60, + fault_ended_at: "not-a-timestamp", + }, + }); + assert.equal(badFaultEndedAt.isError, true); + const badSeverity = await client.callTool({ name: "observe_impact", arguments: { predicted_severity: "catastrophic", affected_routes: ["/products"], window_seconds: 60 }, From 2c9d31263ee64c42e8897aab81fa2cc171c6c6e5 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 11:47:24 +0530 Subject: [PATCH 20/21] test: add short-fault and checkout-api-outage acceptance cases to verify-m5.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add extract_expires_at(), following this script's established convention of piping raw MCP tool JSON responses through python3 on stdin rather than shell-interpolating JSON, to pull expiresAt out of a fault-injection tool's response. New case 1: pauses chaos-pg-replica for a SHORT 12s fault (well under the existing 90s test's 60% fault-fraction), waits well past its own auto-revert plus a real extra delay, then calls observe_impact with fault_ended_at set to the captured expiresAt and window_seconds equal to the fault's own exact duration_seconds — asserts observedSeverity stays "hard"/"matched" despite the real elapsed delay. This is the regression case the prior fixed +60s buffer could not have passed for a fault this short (its own regression test only covered a 90s fault). New case 2: pauses chaos-checkout-api itself (a full API outage, not just a DB dependency) and verifies observe_impact reports "hard" for the routes M4's topology predicts as hard-affected (/products, /orders, /health), even though checkout-api's own request counters cannot increment while the process is paused — proving the up{job="checkout-api"} scrape-health escalation actually works. Existing baseline/mid-fault/buffered-window assertions are unchanged; they still validate the no-fault_ended_at fallback path. Docker is unavailable in this environment, so this script could not be run live as part of this fix — bash -n syntax-checked only. --- scripts/verify-m5.sh | 92 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 4 deletions(-) diff --git a/scripts/verify-m5.sh b/scripts/verify-m5.sh index 06adff4..e175c33 100755 --- a/scripts/verify-m5.sh +++ b/scripts/verify-m5.sh @@ -4,10 +4,15 @@ # # Verifies observe_impact reports accurate real-world severity against # live Prometheus data: "none" at baseline, "hard" with a "matched" -# verdict during a real fault, correctly rejects invalid input, and -# confirms metrics-watcher cannot reach mcp-server over the network (the -# same isolation guard M4 added after Qodo caught the equivalent gap for -# blast-radius-sandbox). +# verdict during a real fault, correctly rejects invalid input, confirms +# metrics-watcher cannot reach mcp-server over the network (the same +# isolation guard M4 added after Qodo caught the equivalent gap for +# blast-radius-sandbox), and — the Qodo PR #5 regression coverage — +# confirms fault_ended_at-based offset anchoring holds up for a SHORT fault +# called well after it ended (the exact scenario a fixed +60s buffer gets +# wrong), and confirms a full chaos-checkout-api outage is reported as +# "hard" via the up{job="checkout-api"} scrape-health signal even though +# checkout-api's own request counters can't increment while it's paused. set -euo pipefail API_URL="http://localhost:3000" @@ -26,6 +31,22 @@ call_metrics_watcher() { docker compose exec -T metrics-watcher node dist/cliCall.js "$1" "$args" } +# Pulls the expiresAt field out of a fault-injection tool's JSON response +# (pause_container/stop_container/kill_container/inject_latency/ +# inject_packet_loss all return { container, expiresAt }). Follows this +# script's own established convention (see the baseline/faulted/delayed +# blocks below) of piping the raw response through python3 on stdin rather +# than interpolating JSON into a shell string, to avoid shell/JSON +# escaping bugs. +extract_expires_at() { + python3 -c " +import json, sys +result = json.load(sys.stdin) +payload = json.loads(result['content'][0]['text']) +print(payload['expiresAt']) +" +} + wait_for() { local desc="$1" cmd="$2" tries="${3:-30}" for _ in $(seq 1 "$tries"); do @@ -109,6 +130,69 @@ assert observation['verdict'] == 'matched', f\"expected matched, got {observatio print('OK: buffered post-revert-delay observe_impact still reports hard/matched') " +echo "== short fault + fault_ended_at: offset-anchored window still reports hard/matched (Qodo #5 regression) ==" +# This is the exact scenario the old fixed +60s buffer got wrong: a SHORT +# fault (12s, well under the 90s test above) is only a small fraction of a +# buffered window, diluting the aggregate error rate below the "hard" +# threshold even though the fault itself was working correctly. Anchoring +# the query window to the fault's own real expiresAt (rather than guessing +# with a buffer) fixes this for any duration, not just long ones. We +# deliberately wait well past the fault's own auto-revert before calling +# observe_impact, to simulate a real "checked back in a while later" call — +# window_seconds is set to exactly the fault's own duration_seconds (12), +# not duration+60. +short_duration=12 +pause_resp=$(call_mcp pause_container "{\"container\":\"chaos-pg-replica\",\"duration_seconds\":${short_duration}}") +echo "$pause_resp" +short_expires_at=$(printf '%s' "$pause_resp" | extract_expires_at) +echo "short fault expiresAt: ${short_expires_at}" + +# Sleep past the fault's own duration plus a real extra delay beyond +# expiresAt, so the offset the server computes at call time is genuinely +# larger than window_seconds — exactly the case a fixed buffer can't +# handle for a short fault. +sleep $((short_duration + 20)) + +short_observe_args=$(printf '{"predicted_severity":"hard","affected_routes":["/products"],"window_seconds":%d,"fault_ended_at":"%s"}' "$short_duration" "$short_expires_at") +short_result=$(call_metrics_watcher observe_impact "$short_observe_args") +echo "$short_result" +printf '%s' "$short_result" | python3 -c " +import json, sys +result = json.load(sys.stdin) +observation = json.loads(result['content'][0]['text']) +assert observation['observedSeverity'] == 'hard', f\"expected hard, got {observation['observedSeverity']}\" +assert observation['verdict'] == 'matched', f\"expected matched, got {observation['verdict']}\" +print('OK: short-fault, offset-anchored observe_impact (called well after the fault ended) still reports hard/matched') +" + +echo "== chaos-checkout-api outage: observedSeverity hard via up{} scrape-health, not request counters (Qodo #5 regression) ==" +# checkout-api's only request counter increments in Fastify's onResponse +# hook, which cannot fire while the process itself is paused — so during +# this fault, checkout-api's own /metrics endpoint is entirely unscrapeable +# and queryRouteMetrics would see no request data at all. Without the +# up{job="checkout-api"} scrape-health check, that null data reads as +# "none" (idle), which is the opposite of correct for the single most +# common chaos scenario. affected_routes below are the routes M4's own +# topology predicts as hard-affected for chaos-checkout-api. +outage_duration=15 +outage_resp=$(call_mcp pause_container "{\"container\":\"chaos-checkout-api\",\"duration_seconds\":${outage_duration}}") +echo "$outage_resp" +outage_expires_at=$(printf '%s' "$outage_resp" | extract_expires_at) +echo "checkout-api outage expiresAt: ${outage_expires_at}" + +sleep $((outage_duration + 10)) + +outage_observe_args=$(printf '{"predicted_severity":"hard","affected_routes":["/products","/orders","/health"],"window_seconds":%d,"fault_ended_at":"%s"}' "$outage_duration" "$outage_expires_at") +outage_result=$(call_metrics_watcher observe_impact "$outage_observe_args") +echo "$outage_result" +printf '%s' "$outage_result" | python3 -c " +import json, sys +result = json.load(sys.stdin) +observation = json.loads(result['content'][0]['text']) +assert observation['observedSeverity'] == 'hard', f\"expected hard, got {observation['observedSeverity']}\" +print('OK: chaos-checkout-api outage correctly reported as hard via up{} scrape-health signal') +" + echo "== network isolation positive control: metrics-watcher CAN reach prometheus ==" # Proves wget works inside the metrics-watcher image and the container can # reach a legitimate target, so the negative-control failure below is From a5f4eb7e80b6bf904e11881b3d0724a2e10bad9f Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 28 Aug 2026 13:25:37 +0530 Subject: [PATCH 21/21] fix: create a fresh MCP transport+server per request in metrics-watcher Identical bug to mcp-server/blast-radius-sandbox (fixed in #6, merged into this branch): one shared McpServer+transport pair at startup made every /mcp request after the first fail with a bare 500, since the installed SDK enforces "one transport per request" in stateless mode. Fixed the same way: a fresh McpServer + StreamableHTTPServerTransport pair per request. Verified against a live stack for the first time ever: bash scripts/verify-m5.sh passes end to end, including the two scenarios Qodo's review specifically flagged as broken (short-fault offset-anchoring via fault_ended_at, and chaos-checkout-api full-outage detection via Prometheus's up{} scrape-health signal) - both previously verified only by code reading and unit tests, now confirmed with real data from a real fault injection. Unit tests unchanged (33/33 pass). --- services/metrics-watcher/src/server.ts | 54 +++++++++++++++----------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/services/metrics-watcher/src/server.ts b/services/metrics-watcher/src/server.ts index 76ae17a..e150f24 100644 --- a/services/metrics-watcher/src/server.ts +++ b/services/metrics-watcher/src/server.ts @@ -6,19 +6,6 @@ import { registerTools } from "./tools.js"; const PORT = Number(process.env.PORT ?? 3300); async function main() { - const mcpServer = new McpServer({ name: "chaos-notary-metrics-watcher", version: "1.0.0" }); - registerTools(mcpServer); - - // Stateless mode: each request is an independent JSON-RPC call. See - // services/mcp-server/src/server.ts (M2) for why — a stateful transport with a generated - // session ID rejects every request after the first client session closes, - // and every caller here (the acceptance script's cliCall.ts) is a - // short-lived, one-call-then-disconnect client. - const transport = new StreamableHTTPServerTransport({ - sessionIdGenerator: undefined, - }); - await mcpServer.connect(transport); - const httpServer = http.createServer((req, res) => { if (req.url === "/health") { res.writeHead(200, { "content-type": "application/json" }); @@ -26,18 +13,41 @@ async function main() { return; } if (req.url === "/mcp") { + // Stateless mode (sessionIdGenerator: undefined): each request is an + // independent JSON-RPC call, and every caller here (the acceptance + // script's cliCall.ts) is a short-lived, one-call-then-disconnect + // client. The SDK enforces "one transport per request" at two levels: + // the transport itself throws "Stateless transport cannot be reused + // across requests" on its second call, and Server.connect() throws + // "Already connected to a transport" if the same McpServer instance + // is still attached to a prior transport — so both the transport AND + // the McpServer must be created fresh per request, not shared for the + // process's whole lifetime (see services/mcp-server/src/server.ts, + // M2, for the same fix and the reasoning/live verification behind + // it — a single shared instance made every request after the first + // fail with a bare 500). registerTools() is cheap (zod schema and a + // closure, no I/O), so re-registering per request costs nothing + // meaningful. + const mcpServer = new McpServer({ name: "chaos-notary-metrics-watcher", version: "1.0.0" }); + registerTools(mcpServer); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + }); // .catch(), not `void` alone: an unhandled rejection here would crash // the whole process on a single malformed request (this exact bug // was found and fixed in M2's server.ts during its Qodo review). - transport.handleRequest(req, res).catch((err) => { - console.error("error handling /mcp request", err); - if (!res.headersSent) { - res.writeHead(500, { "content-type": "application/json" }); - res.end(JSON.stringify({ error: "internal server error" })); - } else { - res.end(); - } - }); + mcpServer + .connect(transport) + .then(() => transport.handleRequest(req, res)) + .catch((err) => { + console.error("error handling /mcp request", err); + if (!res.headersSent) { + res.writeHead(500, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "internal server error" })); + } else { + res.end(); + } + }); return; } res.writeHead(404);