From a81f4814c808238055d45af4bb7e5769e81fe107 Mon Sep 17 00:00:00 2001 From: Wilfredo Concepcion Date: Sat, 4 Jul 2026 23:14:09 -0700 Subject: [PATCH 1/2] Add threaded download monitoring with Prometheus metrics --- modules/download_monitor.py | 56 +++++++++++++++++++++++++++++++++++++ server.py | 7 +++++ 2 files changed, 63 insertions(+) create mode 100644 modules/download_monitor.py diff --git a/modules/download_monitor.py b/modules/download_monitor.py new file mode 100644 index 0000000..1dc01ac --- /dev/null +++ b/modules/download_monitor.py @@ -0,0 +1,56 @@ +import logging +import threading +import time + +from pytubefix import YouTube +from modules.metrics import MetricsHandler + + +TEST_VIDEO_URL = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + + +def monitor_download(): + start = time.time() + + try: + video = YouTube(TEST_VIDEO_URL) + + stream = ( + video.streams.filter( + resolution="360p", + progressive=True, + ) + .order_by("resolution") + .desc() + .first() + ) + + if stream is None: + raise Exception("No suitable stream found") + + stream.download( + output_path="/dev", + filename="null", + ) + + duration = time.time() - start + + MetricsHandler.download_monitor_success.set(1) + MetricsHandler.download_monitor_duration_seconds.set(duration) + + logging.info(f"Download monitoring succeeded in {duration:.2f} seconds") + + except Exception as e: + MetricsHandler.download_monitor_success.set(0) + MetricsHandler.download_monitor_failures_total.inc() + logging.exception(f"Download monitoring failed: {e}") + + +def start_download_monitor(interval: int): + def monitor_loop(): + while True: + monitor_download() + time.sleep(interval) + + thread = threading.Thread(target=monitor_loop, daemon=True) + thread.start() \ No newline at end of file diff --git a/server.py b/server.py index 775f8bd..575041f 100644 --- a/server.py +++ b/server.py @@ -29,6 +29,7 @@ from modules.args import get_args from modules.cache import Cache from modules.metrics import MetricsHandler +from modules.download_monitor import start_download_monitor logging.Formatter.converter = time.gmtime @@ -798,7 +799,13 @@ def startup(): threading.Thread(target=download_video_worker, daemon=True).start() threading.Thread(target=play_video_worker, daemon=True).start() + if args.download_monitoring_interval > 0: + start_download_monitor(args.download_monitoring_interval) + logging.info( + f"Started download monitoring every {args.download_monitoring_interval} seconds" + ) + @app.get("/announcement") async def announcement(): return FileResponse("static/announcement.html") From 8d95b62456674db51946b2f6c07bc267defad363 Mon Sep 17 00:00:00 2001 From: Wilfredo Concepcion Date: Mon, 20 Jul 2026 09:25:07 -0700 Subject: [PATCH 2/2] Add Grafana dashboards for download bitrate metrics --- docker-compose.dev.yml | 18 ++++++++++++++ docker-compose.yml | 2 +- modules/args.py | 6 +++++ modules/download_monitor.py | 42 ++++++++++++++++++++++++++------- modules/metrics.py | 47 ++++++++++++++++++++++++++++++------- 5 files changed, 98 insertions(+), 17 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e957620..83f6957 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,10 +1,12 @@ version: '2' + services: streaming-server: image: 'illuspas/node-media-server' container_name: nms ports: - '1935:1935' + app: build: context: . @@ -12,6 +14,7 @@ services: command: - --videopath=/tmp/videos - --rtmp-stream-url=rtmp://nms:1935/live/mystream + - --download-monitoring-interval=30 # uncomment the below to test an interlude. a file called # interlude.mp4 must exist in this project in the `videos` folder. # there is an unresolved bug where the server doesn't reload @@ -26,6 +29,7 @@ services: - ./videos:/tmp/videos:rw environment: - WATCHFILES_FORCE_POLLING=true + prometheus: image: prom/prometheus:latest restart: always @@ -35,3 +39,17 @@ services: - --config.file=/etc/prometheus/prometheus.yml ports: - 9090:9090 + + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: always + ports: + - "3000:3000" + depends_on: + - prometheus + volumes: + - grafana-data:/var/lib/grafana + +volumes: + grafana-data: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index fdbcaeb..4379f09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: - ./static/:/app/static/ - ./modules:/app/modules - ./videos:/tmp/videos:rw - + networks: default: external: diff --git a/modules/args.py b/modules/args.py index 43883a8..4e5d63f 100644 --- a/modules/args.py +++ b/modules/args.py @@ -31,4 +31,10 @@ def get_args(): "--cache-state-file", help="JSON file to persist cache state on server shutdown and recover on startup. if specified, the server will not empty the cache on shutdown" ) + parser.add_argument( + "--download-monitoring-interval", + type=int, + default=0, + help="Interval in seconds between test downloads. Disabled when set to 0.", + ) return parser.parse_args() diff --git a/modules/download_monitor.py b/modules/download_monitor.py index 1dc01ac..6631681 100644 --- a/modules/download_monitor.py +++ b/modules/download_monitor.py @@ -3,13 +3,14 @@ import time from pytubefix import YouTube + from modules.metrics import MetricsHandler TEST_VIDEO_URL = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" -def monitor_download(): +def monitor_download() -> None: start = time.time() try: @@ -26,7 +27,12 @@ def monitor_download(): ) if stream is None: - raise Exception("No suitable stream found") + raise RuntimeError("No suitable stream found") + + bytes_downloaded = stream.filesize + + if bytes_downloaded is None: + raise RuntimeError("Unable to determine stream file size") stream.download( output_path="/dev", @@ -35,22 +41,42 @@ def monitor_download(): duration = time.time() - start + if duration <= 0: + raise RuntimeError("Download duration must be greater than zero") + + bitrate = bytes_downloaded / duration + MetricsHandler.download_monitor_success.set(1) MetricsHandler.download_monitor_duration_seconds.set(duration) + MetricsHandler.download_bitrate_latest_bytes_per_second.set(bitrate) + MetricsHandler.download_bitrate_bytes_per_second.observe(bitrate) + + logging.info( + "Download monitoring succeeded in %.2f seconds " + "with bitrate %.2f bytes/sec", + duration, + bitrate, + ) - logging.info(f"Download monitoring succeeded in {duration:.2f} seconds") + except Exception as error: + duration = time.time() - start - except Exception as e: MetricsHandler.download_monitor_success.set(0) + MetricsHandler.download_monitor_duration_seconds.set(duration) MetricsHandler.download_monitor_failures_total.inc() - logging.exception(f"Download monitoring failed: {e}") + + logging.exception("Download monitoring failed: %s", error) -def start_download_monitor(interval: int): - def monitor_loop(): +def start_download_monitor(interval: int) -> None: + def monitor_loop() -> None: while True: monitor_download() time.sleep(interval) - thread = threading.Thread(target=monitor_loop, daemon=True) + thread = threading.Thread( + target=monitor_loop, + daemon=True, + name="download-monitor", + ) thread.start() \ No newline at end of file diff --git a/modules/metrics.py b/modules/metrics.py index 20fbd8c..9390393 100644 --- a/modules/metrics.py +++ b/modules/metrics.py @@ -21,7 +21,7 @@ class Metrics(enum.Enum): "subprocess_count", "Number of subprocesses ended", prometheus_client.Counter, - ["exit_code"], # 0, 137, 1 etc + ["exit_code"], # 0, 137, 1 etc. ) DOWNLOAD_TIME = ( @@ -75,15 +75,44 @@ class Metrics(enum.Enum): STREAM_STATE = ( "stream_state", - "Indicates whether the given stream type is running (1=running, 0=stopped)", + "Indicates whether the given stream type is running " + "(1=running, 0=stopped)", prometheus_client.Gauge, ["video_type"], ) + DOWNLOAD_MONITOR_SUCCESS = ( + "download_monitor_success", + "Whether the most recent monitoring download succeeded", + prometheus_client.Gauge, + ) + + DOWNLOAD_MONITOR_DURATION_SECONDS = ( + "download_monitor_duration_seconds", + "Duration of the most recent monitoring download in seconds", + prometheus_client.Gauge, + ) + + DOWNLOAD_MONITOR_FAILURES_TOTAL = ( + "download_monitor_failures_total", + "Total number of failed monitoring downloads", + prometheus_client.Counter, + ) + + DOWNLOAD_BITRATE = ( + "download_bitrate_bytes_per_second", + "Observed bitrate of the 360p test video download in bytes per second", + prometheus_client.Histogram, + ) + + DOWNLOAD_BITRATE_LATEST = ( + "download_bitrate_latest_bytes_per_second", + "Bitrate of the most recent 360p test video download " + "in bytes per second", + prometheus_client.Gauge, + ) + def __init__(self, title, description, prometheus_type, labels=()): - # we use the above default value for labels because it matches what's used - # in the prometheus_client library's metrics constructor, see - # https://github.com/prometheus/client_python/blob/fd4da6cde36a1c278070cf18b4b9f72956774b05/prometheus_client/metrics.py#L115 self.title = title self.description = description self.prometheus_type = prometheus_type @@ -92,12 +121,14 @@ def __init__(self, title, description, prometheus_type, labels=()): class MetricsHandler: @classmethod - def init(self) -> None: + def init(cls) -> None: for metric in Metrics: setattr( - self, + cls, metric.title, metric.prometheus_type( - metric.title, metric.description, labelnames=metric.labels + metric.title, + metric.description, + labelnames=metric.labels, ), )