Services and prometheus

.config/temboard.conf

@@ -49,7 +49,7 @@ method = stderr
 level = DEBUG
 
 [monitoring]
-prometheus = dev/bin/prometheus
+prometheus = ui/build/bin/prometheus
 
 [notifications]
 # SMTP host

Makefile

@@ -15,7 +15,7 @@ develop-%:: .env
 	@dev/bin/checkdocker $(DOCKER_MAX_VERSION)
 	git config blame.ignoreRevsFile .git-blame-ignore-revs
 	if [ -d ~/.config/lnav/formats ] ; then ln -fsTv $$PWD/dev/lnav/formats ~/.config/lnav/formats/temboard ; fi
-	$(MAKE) -j 2 install-$* dev/bin/prometheus
+	$(MAKE) -j 2 install-$* ui/build/bin/prometheus ui/build/bin/promtool
 	mkdir -p dev/temboard
 	cd ui/; npm install-clean
 	cd ui/; npm run build
@@ -56,20 +56,21 @@ install-%: venv-%
 	dev/venv-py$*/bin/temboard-agent --version  # smoke test
 
 # LTS
-PROMETHEUS_VERSION=2.45.1
+PROMETHEUS_VERSION=2.53.0
 dev/downloads/prometheus-%.linux-amd64.tar.gz:
 	mkdir -p $(dir $@)
 	curl --fail --silent -L "https://github.com/prometheus/prometheus/releases/download/v$*/$(notdir $@)" --output $@
 
-dev/bin/prometheus dev/bin/promtool: dev/downloads/prometheus-$(PROMETHEUS_VERSION).linux-amd64.tar.gz
+ui/build/bin/prometheus ui/build/bin/promtool: dev/downloads/prometheus-$(PROMETHEUS_VERSION).linux-amd64.tar.gz
+	mkdir -p $(dir $@)
 	tar --extract --file "$<" --directory "$(dir $@)" --strip-component=1 --touch "prometheus-$(PROMETHEUS_VERSION).linux-amd64/$(notdir $@)"
 	"$@" --version  # Smoketest
 
 clean:  #: Trash venv and containers.
 	docker compose down --volumes --remove-orphans
 	docker rmi --force dalibo/temboard-agent:dev
 	rm -rf dev/venv-py* .venv-py* dev/build/ dev/prometheus/targets/temboard-dev.yaml
-	rm -vf dev/bin/prometheus dev/bin/promtool
+	rm -vf ui/build/bin/prometheus ui/build/bin/promtool
 	rm -rf agent/build/ .env agent/.coverage
 	rm -rvf ui/build/ ui/.coverage
 	$(MAKE) clean-static

agent/temboardagent/web/app.py

@@ -16,6 +16,7 @@
 )
 
 from .. import notification
+from ..toolkit.errors import TemboardError
 from ..toolkit.http import format_date
 from ..toolkit.signing import InvalidSignature, canonicalize_request, verify_v1
 from ..toolkit.utils import JSONEncoder, utcnow
@@ -154,12 +155,17 @@ def wrapper(*a, **kw):
     def authenticate(self):
         app = default_app().temboard
 
-        date = request.headers["x-temboard-date"]
+        date = request.headers.get("x-temboard-date")
+        if not date:
+            raise HTTPError(400, "Missing X-TemBoard-Date header.")
+
         oldest_date = format_date(utcnow() - timedelta(hours=2))
         if date < oldest_date:
             raise HTTPError(400, "Request older than 2 hours.")
 
-        signature = request.headers["x-temboard-signature"]
+        signature = request.headers.get("x-temboard-signature")
+        if not signature:
+            raise HTTPError(400, "Missing X-TemBoard-Signature header.")
         version, _, signature = signature.partition(":")
         if "v1" != version:
             raise HTTPError(400, "Unsupported signature format")
@@ -170,9 +176,12 @@ def authenticate(self):
         path = request.environ["RAW_PATH_INFO"]
         if request.environ["QUERY_STRING"]:
             path = path + "?" + request.environ["QUERY_STRING"]
-        canonical_request = canonicalize_request(
-            request.method, path, request.headers, request.body.read()
-        )
+        try:
+            canonical_request = canonicalize_request(
+                request.method, path, request.headers, request.body.read()
+            )
+        except TemboardError as e:
+            raise HTTPError(400, str(e))
 
         try:
             verify_v1(app.config.signing_key, signature, canonical_request)

agent/temboardagent/web/service.py

@@ -22,7 +22,7 @@ def __str__(self):
         return self.name
 
     # for services.run
-    def setup(self):
+    def setup(self, *_, **__):
         ServerHandler.server_software = "temBoard-agent/%s" % __version__
 
         try:

ui/temboardui/cli/apikey.py

@@ -1,7 +1,5 @@
 import logging
-import string
 import sys
-from secrets import choice
 from textwrap import dedent
 
 from ..model import Session
@@ -31,7 +29,7 @@ def define_arguments(self, parser):
     def main(self, args):
         session = Session()
         key = (
-            ApiKeys.insert(secret=generate_secret(), comment=args.comment)
+            ApiKeys.insert(secret=ApiKeys.generate_secret(), comment=args.comment)
             .with_session(session)
             .scalar()
         )
@@ -101,10 +99,3 @@ def main(self, args):
             logger.info("Purged %d keys.", count)
         else:
             logger.info("No expired keys to purge.")
-
-
-_SECRET_LETTERS = string.ascii_letters + string.digits + "+/-_"
-
-
-def generate_secret(length=40):
-    return "".join(choice(_SECRET_LETTERS) for _ in range(length))

ui/temboardui/cli/app.py

@@ -270,7 +270,6 @@ def __init__(self, app):
         self.app = app
         # Ref to services.BackgroundManager
         self.background = None
-        # For services.run()
         self.perf = perf.PerfCounters.setup(service=self.name)
 
     def __str__(self):
@@ -280,8 +279,10 @@ def __str__(self):
     def create_loop(self):
         return tornado.ioloop.IOLoop.instance()
 
-    def setup(self):
+    def setup(self, sgm, bg):
+        self.background = bg
         if self.perf:
+            sgm.register(self.perf)
             self.perf.run()
 
         flask_app.vitejs.read_manifest()
@@ -347,9 +348,10 @@ def _setup_autoreload(self):
             autoreload.watch(path)
 
     def _autoreload_hook(self):
-        if self.background:
-            logger.debug("Stopping background service before reloading.")
-            self.background.stop()
+        if not self.background:
+            return
+        logger.debug("Stopping background service before reloading.")
+        self.background.stop()
 
     def _iter_template_files(self):
         rootpkg = __import__(__name__)

ui/temboardui/model/orm.py

@@ -1,4 +1,6 @@
 import datetime
+import string
+from secrets import choice
 
 from past.builtins import basestring
 from past.utils import old_div
@@ -137,6 +139,12 @@ class ApiKeys(Model):
     cdate = Column(TIMESTAMP(timezone=True))
     edate = Column(TIMESTAMP(timezone=True))
 
+    _SECRET_LETTERS = string.ascii_letters + string.digits + "+/-_"
+
+    @classmethod
+    def generate_secret(cls, length=40):
+        return "".join(choice(cls._SECRET_LETTERS) for _ in range(length))
+
     # See
     # https://docs.sqlalchemy.org/en/14/orm/queryguide.html#getting-orm-results-from-textual-and-core-statements
 

ui/temboardui/toolkit/services.py

@@ -42,7 +42,7 @@
 
 
 def run(main, *backgrounds):
-    """Execute a main service object function in this process.
+    """Run a main service object function in this process.
 
     Handle signals for INT, TERM, HUP, CHLD and ALRM.
     Handle background services.
@@ -61,17 +61,13 @@ def run(main, *backgrounds):
     sgm.register(LoopStopper(loop))
     sgm.register(main)
 
-    # for tornado autoreload. See TornadoService._autoreload_hook.
-    main.background = bg = BackgroundManager(loop)
+    bg = BackgroundManager(loop)
     if backgrounds:
         for service in backgrounds:
             bg.add(service)
         sgm.register(bg)
 
-    if getattr(main, "perf", None):
-        sgm.register(main.perf)
-
-    main.setup()
+    main.setup(sgm, bg)
     with sgm, bg:
         logger.debug("Entering %s loop.", main)
         loop.start()
@@ -81,6 +77,25 @@ def run(main, *backgrounds):
     return 0
 
 
+def execute(service):
+    """Execute an external command in this process.
+
+    Replace current Python program by a service.command.
+    Does not return. service.command continues process life.
+    """
+
+    if hasattr(service, "setup"):
+        service.setup()
+
+    # Close all files except stdin, stdout, stderr.
+    for fd in range(3, os.sysconf("SC_OPEN_MAX")):
+        try:
+            os.close(fd)
+        except OSError:
+            pass
+    os.execvp(service.command[0], service.command)
+
+
 class LoopStopper:
     def __init__(self, loop):
         self.loop = loop
@@ -103,21 +118,35 @@ def sigterm_handler(self, *a):
 
 class BackgroundManager:
     def __init__(self, loop):
+        self.loop = loop
         self.services = {}
         self.pids = {}
         self.stopping = False  # Whether to restart on SIGCHLD
 
     def add(self, service):
         self.services[str(service)] = service
 
+    def __bool__(self):
+        return bool(self.services)
+
     def __enter__(self):
         if not self.services:
             return
+
+        self._read_pids()
+
+        if self.pids:
+            logger.debug("Cleaning previous background services.")
+            self.kill()
+            if self.wait():
+                raise Exception("Background services are still alive.")
         self.start()
 
-    def __exit__(self, *a):
+    def __exit__(self, etype, evalue, etb):
         if not self.services:
             return
+        if etype:
+            logger.warning("Exiting on error. err=%s", evalue)
         self.stop()
 
     def start(self):
@@ -132,17 +161,54 @@ def fork(self, service):
         pid = os.fork()
         if pid:  # Parent process
             logger.debug("Background service started. service=%s pid=%d", service, pid)
-            self.pids[str(service)] = pid
+            self._save_pid(service, pid)
             return pid
 
         # Child process
-        os._exit(run(service))
+        if hasattr(self.loop, "asyncio_loop"):
+            # Cleanup parent signals handling.
+            # See https://bugs.python.org/issue22087 and https://bugs.python.org/issue21998 for details about asyncio and fork.
+            signal.set_wakeup_fd(-1)
+
+        if hasattr(service, "command"):
+            execute(service)
+        else:
+            os._exit(run(service))
+
+    def _read_pids(self):
+        for name, service in self.services.items():
+            if not hasattr(service, "pidfile"):
+                continue
+            if not os.path.exists(service.pidfile):
+                continue
+            with open(service.pidfile) as fo:
+                self.pids[name] = int(fo.read().strip())
+                logger.debug(
+                    "Read pid from pidfile. service=%s pid=%d", name, self.pids[name]
+                )
+
+    def _save_pid(self, service, pid):
+        self.pids[str(service)] = pid
+        if hasattr(service, "pidfile"):
+            with open(service.pidfile, "w") as fo:
+                fo.write(str(pid))
+
+    def _drop_pid(self, name):
+        del self.pids[name]
+        s = self.services[name]
+        if not hasattr(s, "pidfile"):
+            return
+        if os.path.exists(s.pidfile):
+            os.unlink(s.pidfile)
 
     def stop(self):
         self.stopping = True
         for name, pid in self.pids.items():
             logger.debug("Terminating background service. service=%s pid=%d", name, pid)
-            os.kill(pid, signal.SIGTERM)
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
         time.sleep(0.125)
         if self.wait():
             self.kill()
@@ -160,20 +226,25 @@ def wait(self, timeout=5, step=0.5):
                 try:
                     pid, status = os.waitpid(-1, os.WNOHANG)
                 except ChildProcessError:
-                    break
+                    pass
                 if pid:
-                    del self.pids[name]
+                    self._drop_pid(name)
 
             time.sleep(step)
             timeout -= step
 
         return bool(self.pids)
 
-    def kill(self):
+    def kill(self, sig=signal.SIGKILL):
         for name, pid in self.pids.items():
-            logger.warning("Killing background service. service=%s pid=%s", name, pid)
+            logger.warning(
+                "Signaling background service. service=%s pid=%s signal=%s",
+                name,
+                pid,
+                sig,
+            )
             try:
-                os.kill(pid, signal.SIGKILL)
+                os.kill(pid, sig)
             except ProcessLookupError:
                 pass
 
@@ -195,11 +266,9 @@ def sigchld_handler(self, *a):
                 pass
 
             logger.warning(
-                "Background service dead. Restarting. service=%s pid=%s",
-                name,
-                self.pids[name],
+                "Background service dead. Restarting. service=%s pid=%s", name, pid
             )
-            del self.pids[name]
+            self._drop_pid(name)
 
         self.start()
 

ui/temboardui/toolkit/syncio.py

@@ -74,8 +74,6 @@ class SignalManager(dict):
     # Context manager for synchronous signal handling.
 
     def __enter__(self):
-        # forking from asyncio loop requires reset of wakeup_fd.
-        signal.set_wakeup_fd(-1)
         for sig, handler in self.items():
             signal.signal(sig, handler)
         self._registered = self.keys()