Add Grafana Dashboards for SCE TV Download Metrics

docker-compose.dev.yml

@@ -1,17 +1,20 @@
 version: '2'
+
 services:
   streaming-server:
     image: 'illuspas/node-media-server'
     container_name: nms
     ports:
       - '1935:1935'
+
   app:
     build:
       context: .
       dockerfile: Dockerfile
     command:
       - --videopath=/tmp/videos
       - --rtmp-stream-url=rtmp://nms:1935/live/mystream
+      - --download-monitoring-interval=30
       # uncomment the below to test an interlude. a file called
       # interlude.mp4 must exist in this project in the `videos` folder.
       # there is an unresolved bug where the server doesn't reload
@@ -26,6 +29,7 @@ services:
       - ./videos:/tmp/videos:rw
     environment:
       - WATCHFILES_FORCE_POLLING=true
+
   prometheus:
     image: prom/prometheus:latest
     restart: always
@@ -35,3 +39,17 @@ services:
       - --config.file=/etc/prometheus/prometheus.yml
     ports:
       - 9090:9090
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    restart: always
+    ports:
+      - "3000:3000"
+    depends_on:
+      - prometheus
+    volumes:
+      - grafana-data:/var/lib/grafana
+
+volumes:
+  grafana-data:

docker-compose.yml

@@ -22,7 +22,7 @@ services:
       - ./static/:/app/static/
       - ./modules:/app/modules
       - ./videos:/tmp/videos:rw
-
+  
 networks: 
   default:
     external: 

modules/args.py

@@ -31,4 +31,10 @@ def get_args():
         "--cache-state-file",
         help="JSON file to persist cache state on server shutdown and recover on startup. if specified, the server will not empty the cache on shutdown"
     )
+    parser.add_argument(
+    "--download-monitoring-interval",
+    type=int,
+    default=0,
+    help="Interval in seconds between test downloads. Disabled when set to 0.",
+    )
     return parser.parse_args()

modules/download_monitor.py

@@ -0,0 +1,82 @@
+import logging
+import threading
+import time
+
+from pytubefix import YouTube
+
+from modules.metrics import MetricsHandler
+
+
+TEST_VIDEO_URL = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
+
+
+def monitor_download() -> None:
+    start = time.time()
+
+    try:
+        video = YouTube(TEST_VIDEO_URL)
+
+        stream = (
+            video.streams.filter(
+                resolution="360p",
+                progressive=True,
+            )
+            .order_by("resolution")
+            .desc()
+            .first()
+        )
+
+        if stream is None:
+            raise RuntimeError("No suitable stream found")
+
+        bytes_downloaded = stream.filesize
+
+        if bytes_downloaded is None:
+            raise RuntimeError("Unable to determine stream file size")
+
+        stream.download(
+            output_path="/dev",
+            filename="null",
+        )
+
+        duration = time.time() - start
+
+        if duration <= 0:
+            raise RuntimeError("Download duration must be greater than zero")
+
+        bitrate = bytes_downloaded / duration
+
+        MetricsHandler.download_monitor_success.set(1)
+        MetricsHandler.download_monitor_duration_seconds.set(duration)
+        MetricsHandler.download_bitrate_latest_bytes_per_second.set(bitrate)
+        MetricsHandler.download_bitrate_bytes_per_second.observe(bitrate)
+
+        logging.info(
+            "Download monitoring succeeded in %.2f seconds "
+            "with bitrate %.2f bytes/sec",
+            duration,
+            bitrate,
+        )
+
+    except Exception as error:
+        duration = time.time() - start
+
+        MetricsHandler.download_monitor_success.set(0)
+        MetricsHandler.download_monitor_duration_seconds.set(duration)
+        MetricsHandler.download_monitor_failures_total.inc()
+
+        logging.exception("Download monitoring failed: %s", error)
+
+
+def start_download_monitor(interval: int) -> None:
+    def monitor_loop() -> None:
+        while True:
+            monitor_download()
+            time.sleep(interval)
+
+    thread = threading.Thread(
+        target=monitor_loop,
+        daemon=True,
+        name="download-monitor",
+    )
+    thread.start()

modules/metrics.py

@@ -21,7 +21,7 @@ class Metrics(enum.Enum):
         "subprocess_count",
         "Number of subprocesses ended",
         prometheus_client.Counter,
-        ["exit_code"],  # 0, 137, 1 etc
+        ["exit_code"],  # 0, 137, 1 etc.
     )
 
     DOWNLOAD_TIME = (
@@ -75,15 +75,44 @@ class Metrics(enum.Enum):
 
     STREAM_STATE = (
         "stream_state",
-        "Indicates whether the given stream type is running (1=running, 0=stopped)",
+        "Indicates whether the given stream type is running "
+        "(1=running, 0=stopped)",
         prometheus_client.Gauge,
         ["video_type"],
     )
 
+    DOWNLOAD_MONITOR_SUCCESS = (
+        "download_monitor_success",
+        "Whether the most recent monitoring download succeeded",
+        prometheus_client.Gauge,
+    )
+
+    DOWNLOAD_MONITOR_DURATION_SECONDS = (
+        "download_monitor_duration_seconds",
+        "Duration of the most recent monitoring download in seconds",
+        prometheus_client.Gauge,
+    )
+
+    DOWNLOAD_MONITOR_FAILURES_TOTAL = (
+        "download_monitor_failures_total",
+        "Total number of failed monitoring downloads",
+        prometheus_client.Counter,
+    )
+
+    DOWNLOAD_BITRATE = (
+        "download_bitrate_bytes_per_second",
+        "Observed bitrate of the 360p test video download in bytes per second",
+        prometheus_client.Histogram,
+    )
+
+    DOWNLOAD_BITRATE_LATEST = (
+        "download_bitrate_latest_bytes_per_second",
+        "Bitrate of the most recent 360p test video download "
+        "in bytes per second",
+        prometheus_client.Gauge,
+    )
+
     def __init__(self, title, description, prometheus_type, labels=()):
-        # we use the above default value for labels because it matches what's used
-        # in the prometheus_client library's metrics constructor, see
-        # https://github.com/prometheus/client_python/blob/fd4da6cde36a1c278070cf18b4b9f72956774b05/prometheus_client/metrics.py#L115
         self.title = title
         self.description = description
         self.prometheus_type = prometheus_type
@@ -92,12 +121,14 @@ def __init__(self, title, description, prometheus_type, labels=()):
 
 class MetricsHandler:
     @classmethod
-    def init(self) -> None:
+    def init(cls) -> None:
         for metric in Metrics:
             setattr(
-                self,
+                cls,
                 metric.title,
                 metric.prometheus_type(
-                    metric.title, metric.description, labelnames=metric.labels
+                    metric.title,
+                    metric.description,
+                    labelnames=metric.labels,
                 ),
             )

server.py

@@ -29,6 +29,7 @@
 from modules.args import get_args
 from modules.cache import Cache
 from modules.metrics import MetricsHandler
+from modules.download_monitor import start_download_monitor
 
 
 logging.Formatter.converter = time.gmtime
@@ -798,7 +799,13 @@ def startup():
     threading.Thread(target=download_video_worker, daemon=True).start()
     threading.Thread(target=play_video_worker, daemon=True).start()
 
+    if args.download_monitoring_interval > 0:
+        start_download_monitor(args.download_monitoring_interval)
 
+        logging.info(
+            f"Started download monitoring every {args.download_monitoring_interval} seconds"
+        )
+
 @app.get("/announcement")
 async def announcement():
     return FileResponse("static/announcement.html")