Add maintenance.py for K8s CronJob scheduling

AGENT.md

@@ -24,7 +24,11 @@ Prefer fixup commits over amending and force-pushing.
 
 ## Maintenance Tooling
 
-`tools/justfile` contains DuckLake maintenance recipes (expire snapshots, cleanup files, delete orphans, compaction). It is copied to `/justfile` in the Docker image and uses the pod's existing env vars. The `DUCKDB` env var can override the duckdb binary path.
+`tools/maintenance.py` is a self-contained Python script for DuckLake maintenance operations (snapshot expiry, file cleanup, orphan deletion, checkpoint). It connects to DuckLake using the same env vars as the main app (`DUCKLAKE_RDS_*`, `DUCKDB_S3_*`, `DUCKLAKE_DATA_PATH`) but does not import from the `millpond` package. Designed to run as a K8s CronJob reusing the same Docker image.
+
+If `PUSHGATEWAY_URL` is set, the script pushes two metrics: `maintenance_start_time{operation}` (pushed immediately on start) and `maintenance_duration_seconds{operation, status}` (pushed on completion). This enables Grafana annotation queries for maintenance windows.
+
+`tools/justfile` wraps the script for interactive use and is copied to `/justfile` in the Docker image. The `shell` and `drop` recipes still use the DuckDB CLI directly. The `DUCKDB` env var can override the duckdb binary path.
 
 ## What This Is
 
@@ -375,7 +379,9 @@ millpond/
 │   ├── metrics.py            # Prometheus metric definitions
 │   └── server.py             # HTTP server for /metrics and /healthz
 ├── tools/
-│   └── sizing-calculator.html  # Interactive flush/object sizing calculator
+│   ├── maintenance.py           # Self-contained DuckLake maintenance script (K8s CronJob)
+│   ├── justfile                 # Interactive wrapper for maintenance.py + DuckDB CLI shell
+│   └── sizing-calculator.html   # Interactive flush/object sizing calculator
 ├── tests/
 │   ├── unit/                 # Fast, no external deps
 │   ├── integration/          # Local DuckDB write path + schema evolution

Dockerfile

@@ -23,6 +23,7 @@ COPY --from=builder /app/.venv /app/.venv
 COPY --from=builder /app/millpond /app/millpond
 COPY --from=builder /root/.local/bin/duckdb /usr/local/bin/duckdb
 COPY tools/justfile /justfile
+COPY tools/maintenance.py /app/tools/maintenance.py
 
 ENV PATH="/app/.venv/bin:$PATH"
 

README.md

@@ -83,7 +83,20 @@ Requires Docker (uses `keytool` from the Kafka container image for cert generati
 
 ### DuckLake Maintenance
 
-`tools/justfile` contains DuckLake maintenance recipes and is baked into the Docker image at `/justfile`. From inside a running pod:
+`tools/maintenance.py` is a self-contained Python script for DuckLake maintenance operations (snapshot expiry, file cleanup, orphan deletion, checkpoint). It is baked into the Docker image at `/app/tools/maintenance.py` and designed to run as a K8s CronJob reusing the same image and credentials as the main application.
+
+```bash
+python /app/tools/maintenance.py maintain --days 7          # expire snapshots + cleanup files
+python /app/tools/maintenance.py maintain --days 7 --dry-run # preview only
+python /app/tools/maintenance.py expire --days 3            # expire snapshots only
+python /app/tools/maintenance.py cleanup --days 1           # cleanup scheduled files only
+python /app/tools/maintenance.py checkpoint                 # integrated merge + expire + cleanup
+python /app/tools/maintenance.py orphans                    # delete orphaned S3 files
+```
+
+If `PUSHGATEWAY_URL` is set, the script pushes `maintenance_start_time` (on start) and `maintenance_duration_seconds` (on completion) to a Prometheus Pushgateway, enabling Grafana annotation queries for maintenance windows.
+
+`tools/justfile` wraps the script and is also baked into the image at `/justfile` for interactive use:
 
 ```bash
 just --list              # see available recipes
@@ -94,7 +107,7 @@ just drop events         # drop a table (data files remain until cleanup)
 just orphans-dry-run     # preview orphaned S3 files
 ```
 
-All recipes use the pod's existing env vars (`DUCKLAKE_RDS_*`, `DUCKDB_S3_*`, `DUCKLAKE_DATA_PATH`).
+All commands use the pod's existing env vars (`DUCKLAKE_RDS_*`, `DUCKDB_S3_*`, `DUCKLAKE_DATA_PATH`).
 
 ## Configuration
 

tools/justfile

@@ -1,4 +1,5 @@
 # DuckLake maintenance tasks
+# Delegates to maintenance.py for actual operations.
 # Requires: DUCKLAKE_RDS_PASSWORD, DUCKDB_S3_ACCESS_KEY_ID, DUCKDB_S3_SECRET_ACCESS_KEY
 
 duckdb := env("DUCKDB", "duckdb")
@@ -13,7 +14,9 @@ s3_key := env("DUCKDB_S3_ACCESS_KEY_ID", "")
 s3_secret := env("DUCKDB_S3_SECRET_ACCESS_KEY", "")
 data_path := env("DUCKLAKE_DATA_PATH", "")
 
-_setup := "LOAD httpfs; LOAD ducklake; LOAD postgres; SET s3_region = '" + s3_region + "'; SET s3_endpoint = 's3." + s3_region + ".amazonaws.com'; SET s3_access_key_id = '" + s3_key + "'; SET s3_secret_access_key = '" + s3_secret + "'; SET s3_use_ssl = true; SET s3_url_style = 'vhost'; ATTACH 'postgres:host=" + rds_host + " port=" + rds_port + " dbname=" + rds_db + " user=" + rds_user + " password=" + rds_password + "' AS lake (TYPE ducklake, DATA_PATH '" + data_path + "'); USE lake;"
+_setup := "INSTALL httpfs; INSTALL ducklake; INSTALL postgres; LOAD httpfs; LOAD ducklake; LOAD postgres; SET s3_region = '" + s3_region + "'; SET s3_endpoint = 's3." + s3_region + ".amazonaws.com'; SET s3_access_key_id = '" + s3_key + "'; SET s3_secret_access_key = '" + s3_secret + "'; SET s3_use_ssl = true; SET s3_url_style = 'vhost'; ATTACH 'postgres:host=" + rds_host + " port=" + rds_port + " dbname=" + rds_db + " user=" + rds_user + " password=" + rds_password + "' AS lake (TYPE ducklake, DATA_PATH '" + data_path + "'); USE lake;"
+
+maintenance := env("MAINTENANCE_SCRIPT", "/app/tools/maintenance.py")
 
 default:
     @just --list
@@ -28,42 +31,40 @@ drop table:
 
 # Expire snapshots (dry run)
 expire-dry-run days="7":
-    {{ duckdb }} -c "{{ _setup }} CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL '{{ days }} days', dry_run => true);"
+    python {{ maintenance }} expire --days {{ days }} --dry-run
 
 # Expire snapshots
 expire days="7":
-    {{ duckdb }} -c "{{ _setup }} CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL '{{ days }} days');"
+    python {{ maintenance }} expire --days {{ days }}
 
 # Preview files scheduled for deletion
 cleanup-dry-run days="1":
-    {{ duckdb }} -c "{{ _setup }} CALL ducklake_cleanup_old_files('lake', older_than => now() - INTERVAL '{{ days }} days', dry_run => true);"
+    python {{ maintenance }} cleanup --days {{ days }} --dry-run
 
 # Delete files scheduled for deletion
 cleanup days="1":
-    {{ duckdb }} -c "{{ _setup }} CALL ducklake_cleanup_old_files('lake', older_than => now() - INTERVAL '{{ days }} days');"
+    python {{ maintenance }} cleanup --days {{ days }}
 
 # Delete all files scheduled for deletion regardless of age
 cleanup-all:
-    {{ duckdb }} -c "{{ _setup }} CALL ducklake_cleanup_old_files('lake', cleanup_all => true);"
+    python {{ maintenance }} cleanup-all
 
 # Preview orphaned files
 orphans-dry-run:
-    {{ duckdb }} -c "{{ _setup }} CALL ducklake_delete_orphaned_files('lake', dry_run => true);"
+    python {{ maintenance }} orphans --dry-run
 
 # Delete orphaned files
 orphans:
-    {{ duckdb }} -c "{{ _setup }} CALL ducklake_delete_orphaned_files('lake');"
+    python {{ maintenance }} orphans
 
 # Full maintenance: expire snapshots + cleanup files
 maintain days="7":
-    just expire {{ days }}
-    just cleanup {{ days }}
+    python {{ maintenance }} maintain --days {{ days }}
 
 # Full maintenance dry run
 maintain-dry-run days="7":
-    just expire-dry-run {{ days }}
-    just cleanup-dry-run {{ days }}
+    python {{ maintenance }} maintain --days {{ days }} --dry-run
 
 # Run CHECKPOINT (integrated merge + expire + cleanup)
 checkpoint:
-    {{ duckdb }} -c "{{ _setup }} CHECKPOINT lake;"
+    python {{ maintenance }} checkpoint