docker-stack.yml

# Shazamer as a Docker Swarm service (replaces the blue-green standalone).
# Deploy:  set -a; . .env; set +a; docker stack deploy -c docker-stack.yml shazamer
#
# Routed by Traefik via deploy.labels (swarm provider on the `traefik`
# overlay). Host comes from $SHAZAMER_HOST (kept out of the repo, set in
# the gitignored .env). `docker stack deploy` ignores env_file/mem_limit/
# container_name, so SENTRY_DSN is interpolated and the limit is in
# deploy.resources.
services:
  app:
    image: shazamer_app:latest
    environment:
      - PYTHONUNBUFFERED=1
      - PYTHON_ENV=Production
      - SENTRY_DSN=${SENTRY_DSN:-}
      - SHAZAM_CONCURRENCY=${SHAZAM_CONCURRENCY:-4}
      - KEEP_AUDIO_DAYS=${KEEP_AUDIO_DAYS:-0}
      - MIN_FREE_DISK_GB=${MIN_FREE_DISK_GB:-5}
      - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-https://shazamer.pierregallet.com}
      - SLSKD_URL=${SLSKD_URL:-}
      - SLSKD_API_KEY=${SLSKD_API_KEY:-}
      - REDIS_URL=redis://redis:6379
      # Accounts. The worker gets them too: it never serves a request, but it
      # saves sets, and a set has an owner.
      # Where this instance lives, for links that leave the app — a share
      # invitation has to point somewhere reachable. Not derived from the
      # request Host header: trusting that would let anyone who can reach the
      # API mint an invitation pointing at a site they control.
      - PUBLIC_URL=${PUBLIC_URL:-}
      - SMTP_HOST=${SMTP_HOST:-}
      - SMTP_PORT=${SMTP_PORT:-587}
      - SMTP_USER=${SMTP_USER:-}
      - SMTP_PASSWORD=${SMTP_PASSWORD:-}
      - SMTP_SSL=${SMTP_SSL:-}
      - SMTP_STARTTLS=${SMTP_STARTTLS:-1}
      - MAIL_FROM=${MAIL_FROM:-}
      - MAIL_FROM_NAME=${MAIL_FROM_NAME:-Shazamer}
      - COOKIE_SECURE=${COOKIE_SECURE:-1}
      - COOKIE_SAMESITE=${COOKIE_SAMESITE:-lax}
      - COOKIE_DOMAIN=${COOKIE_DOMAIN:-}
      # Documented in .env.example, so they have to reach the container or
      # setting them does nothing and looks like the code ignoring them.
      - MUSICBRAINZ_CONTACT=${MUSICBRAINZ_CONTACT:-}
      - MUSICBRAINZ_INTERVAL=${MUSICBRAINZ_INTERVAL:-1.5}
      - MUSICBRAINZ_MIN_SCORE=${MUSICBRAINZ_MIN_SCORE:-88}
    volumes:
      - type: bind
        source: /home/sharon/shazamer/data
        target: /app/data
      - type: bind
        source: /home/sharon/shazamer/media
        target: /app/media
      - type: bind
        source: /home/sharon/shazamer/uploads
        target: /app/uploads
      - type: bind
        source: /home/sharon/shazamer/tmp
        target: /app/tmp
      - type: bind
        source: /home/sharon/shazamer/downloads
        target: /app/downloads
    networks:
      - traefik
      - internal
    healthcheck:
      # See scripts/healthcheck.py: a raw socket under `python -S`, kept in a
      # file so no CRLF has to survive YAML and a shell. It costs a third of
      # the urllib version it replaced, which timed out under normal load and
      # had Swarm killing healthy containers mid-analysis.
      #
      # The tolerance matches what this service actually does. Analysis is
      # CPU-bound by design, so a probe that is merely slow is not evidence of
      # a sick process — only sustained silence is. 20s x 5 means roughly two
      # minutes of genuine unresponsiveness before the container is replaced.
      test: ["CMD", "python", "-S", "/app/healthcheck.py"]
      interval: 30s
      timeout: 20s
      retries: 5
      start_period: 120s
    deploy:
      replicas: 1
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.shazamer.rule=Host(`${SHAZAMER_HOST:-shazamer.pierregallet.com}`)"
        - "traefik.http.routers.shazamer.entrypoints=web"
        - "traefik.http.services.shazamer.loadbalancer.server.port=8000"
        # Block /metrics from the public internet. The endpoint carries no user
        # data, but it publishes every route name, request volume and latency
        # profile — a free reconnaissance map. A dedicated router with a higher
        # priority wins the match and the allowlist middleware answers 403.
        # Prometheus is unaffected: it scrapes shazamer_app:8000 directly over
        # the Swarm overlay and never traverses Traefik.
        #
        # `@swarm` et NON `@docker` : le suffixe nomme le fournisseur qui a
        # decouvert l'etiquette, et ces etiquettes viennent du fournisseur
        # Swarm. Avec `@docker` le middleware est introuvable, Traefik ecarte le
        # routeur comme invalide, et la requete retombe sur le routeur d'hote
        # qui sert les metriques. Un `curl` public renvoyait 200 et le journal
        # d'acces nommait le routeur d'hote : le blocage n'avait jamais
        # fonctionne, sur aucun des trois depots qui le declarent.
        #
        # priority=1000 et non 100 : une priorite absente vaut la LONGUEUR de la
        # regle, et 100 n'etait pas surement au-dessus du routeur d'hote.
        - "traefik.http.routers.shazamer-metrics.rule=Host(`${SHAZAMER_HOST:-shazamer.pierregallet.com}`) && PathPrefix(`/metrics`)"
        - "traefik.http.routers.shazamer-metrics.entrypoints=web"
        - "traefik.http.routers.shazamer-metrics.priority=1000"
        - "traefik.http.routers.shazamer-metrics.middlewares=shazamer-deny@swarm"
        - "traefik.http.routers.shazamer-metrics.service=shazamer"
        - "traefik.http.middlewares.shazamer-deny.ipallowlist.sourcerange=127.0.0.1/32"
      resources:
        limits:
          # The API no longer analyses anything — that moved to the worker — so
          # it needs room to serve requests and to fall back to running an
          # analysis in-process if Redis is unreachable. 1G covers both.
          memory: 1G
          # Serving requests is close to free; the CPU belongs to the worker.
          cpus: "1.0"
      update_config:
        order: start-first
        failure_action: rollback
      restart_policy:
        # `any`, not `on-failure`. A web service has no successful exit: if the
        # process is gone, the site is down, whatever the exit code says.
        # Under `on-failure` a clean shutdown — a SIGTERM from anywhere, a
        # graceful uvicorn stop during an update — left the task marked
        # "Complete" and Swarm declined to replace it, so the service sat at
        # 0/1 and every request 404'd until someone forced it back.
        condition: any
        delay: 5s
        max_attempts: 0        # keep trying; a permanently broken image is
                               # caught by failure_action: rollback above
        window: 120s

  # The analyses themselves. Same image, same state volume, different job:
  # the API answers requests, this decodes audio for an hour at a time. Split
  # so a long decode can no longer starve the API's health probe — which is
  # how Swarm came to kill a container that was working perfectly.
  worker:
    image: shazamer_app:latest
    command: ["arq", "src.jobs.worker.WorkerSettings"]
    environment:
      - PYTHONUNBUFFERED=1
      - PYTHON_ENV=Production
      - SENTRY_DSN=${SENTRY_DSN:-}
      - SHAZAM_CONCURRENCY=${SHAZAM_CONCURRENCY:-4}
      - KEEP_AUDIO_DAYS=${KEEP_AUDIO_DAYS:-0}
      - MIN_FREE_DISK_GB=${MIN_FREE_DISK_GB:-5}
      - SLSKD_URL=${SLSKD_URL:-}
      - SLSKD_API_KEY=${SLSKD_API_KEY:-}
      - SLSKD_DOWNLOADS_DIR=/slskd-downloads
      - REDIS_URL=redis://redis:6379
      # Accounts. The worker gets them too: it never serves a request, but it
      # saves sets, and a set has an owner.
      # Where this instance lives, for links that leave the app — a share
      # invitation has to point somewhere reachable. Not derived from the
      # request Host header: trusting that would let anyone who can reach the
      # API mint an invitation pointing at a site they control.
      - PUBLIC_URL=${PUBLIC_URL:-}
      - SMTP_HOST=${SMTP_HOST:-}
      - SMTP_PORT=${SMTP_PORT:-587}
      - SMTP_USER=${SMTP_USER:-}
      - SMTP_PASSWORD=${SMTP_PASSWORD:-}
      - SMTP_SSL=${SMTP_SSL:-}
      - SMTP_STARTTLS=${SMTP_STARTTLS:-1}
      - MAIL_FROM=${MAIL_FROM:-}
      - MAIL_FROM_NAME=${MAIL_FROM_NAME:-Shazamer}
      - COOKIE_SECURE=${COOKIE_SECURE:-1}
      - COOKIE_SAMESITE=${COOKIE_SAMESITE:-lax}
      - COOKIE_DOMAIN=${COOKIE_DOMAIN:-}
      # Documented in .env.example, so they have to reach the container or
      # setting them does nothing and looks like the code ignoring them.
      - MUSICBRAINZ_CONTACT=${MUSICBRAINZ_CONTACT:-}
      - MUSICBRAINZ_INTERVAL=${MUSICBRAINZ_INTERVAL:-1.5}
      - MUSICBRAINZ_MIN_SCORE=${MUSICBRAINZ_MIN_SCORE:-88}
    volumes:
      - type: bind
        source: /home/sharon/shazamer/data
        target: /app/data
      - type: bind
        source: /home/sharon/shazamer/media
        target: /app/media
      - type: bind
        source: /home/sharon/shazamer/uploads
        target: /app/uploads
      - type: bind
        source: /home/sharon/shazamer/tmp
        target: /app/tmp
      - type: bind
        source: /home/sharon/shazamer/downloads
        target: /app/downloads
      # Where slskd drops finished transfers. Read-only: slskd owns what it
      # writes, this only collects from it. The path must match
      # SLSKD_DOWNLOADS_DIR or the file is fetched and then lost.
      - type: bind
        source: /home/sharon/slskd/downloads
        target: /slskd-downloads
        read_only: true
    networks:
      - internal
    # Disabled, not replaced. The image's HEALTHCHECK probes an HTTP port that
    # only the API serves, so the worker inherited a check it could never pass
    # and Swarm killed it in a loop — the container was fine, the question was
    # nonsense.
    #
    # Nothing takes its place on purpose. arq reconnects to Redis by itself, a
    # dead process is already caught by the restart policy, and today has been
    # a long lesson in what a health probe that can be wrong costs.
    healthcheck:
      test: ["NONE"]
    deploy:
      # Exactly one. The startup reclaim assumes a worker starting means
      # nothing else is running; with two, a starting worker could snatch a job
      # the other is part-way through. Raising this needs that revisited.
      replicas: 1
      resources:
        limits:
          # Measured at ~550 MB mid-analysis on a 69-minute set: the Python
          # process plus `concurrency` ffmpeg subprocesses. 2G is headroom over
          # a measurement, not over an estimate — an earlier limit set from a
          # tracemalloc figure was an order of magnitude too low.
          memory: 2G
          # Where the work actually happens. 4 of the host's 8 cores — which
          # is a ceiling, not a reservation: this box runs several projects and
          # the analysis gets what is left. Under load it simply takes longer.
          cpus: "4.0"
      restart_policy:
        condition: any
        delay: 5s
        max_attempts: 0
        window: 120s

  # Broker only — task state lives on the shared volume, so there is one source
  # of truth and nothing to keep in sync. Losing Redis costs queued jobs, not
  # results; appendonly keeps even that across a restart.
  redis:
    image: redis:7-alpine
    command: ["redis-server", "--appendonly", "yes", "--save", ""]
    volumes:
      - type: bind
        source: /home/sharon/shazamer/redis
        target: /data
    networks:
      - internal
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 30s
      timeout: 5s
      retries: 3
    deploy:
      replicas: 1
      resources:
        limits:
          memory: 256M
          cpus: "0.5"
      restart_policy:
        condition: any
        delay: 5s

  # The Soulseek client. A separate daemon holding the network connection,
  # driven over its REST API — this app never speaks the protocol itself.
  #
  # Off unless SLSKD_SLSK_USERNAME is set: the service starts either way, but
  # without credentials it cannot log in, and the app hides the feature when
  # SLSKD_URL is unset.
  #
  # Two things about Soulseek that are not optional:
  #
  #   * The network refuses two sessions on one account. Running this while a
  #     desktop client is logged in with the same credentials makes the two
  #     evict each other in a loop. Give the server its own account.
  #   * It must share something. A client that only takes is throttled and
  #     eventually banned — peers check. `shared` below is what it offers, and
  #     it starts empty, so seed it.
  slskd:
    image: slskd/slskd:latest
    environment:
      - SLSKD_REMOTE_CONFIGURATION=false
      - SLSKD_NO_AUTH=false
      # The API key is NOT set here. slskd's environment mapping does not
      # reach dictionary entries, so these variables were accepted silently and
      # registered nothing — every call came back "rejected the API key" while
      # the key itself was correct. deploy.sh writes it into slskd.yml instead.
      - SLSKD_SLSK_USERNAME=${SLSKD_SLSK_USERNAME:-}
      - SLSKD_SLSK_PASSWORD=${SLSKD_SLSK_PASSWORD:-}
      - SLSKD_SHARED_DIR=/shared
      - SLSKD_DOWNLOADS_DIR=/downloads
    volumes:
      - type: bind
        source: /home/sharon/slskd/config
        target: /app
      - type: bind
        source: /home/sharon/slskd/downloads
        target: /downloads
      # What the server offers back: the tracks it has fetched. You share what
      # you take, and it grows with use rather than needing to be curated.
      # Read-only — sharing does not mean letting the network write here.
      #
      # This is why downloads have their own, much longer retention: on the
      # fortnightly schedule that set audio uses, the share would empty itself.
      - type: bind
        source: /home/sharon/shazamer/downloads
        target: /shared
        read_only: true
    networks:
      - internal
    deploy:
      replicas: 1
      resources:
        limits:
          memory: 512M
          # Deliberately fractional. A client that cannot log in retries in a
          # tight loop, and on a shared box that loop took a whole core away
          # from the analysis worker while accomplishing nothing. Capped, the
          # same failure is merely slow instead of expensive.
          cpus: "0.5"
      restart_policy:
        condition: any
        delay: 10s

networks:
  traefik:
    external: true
  internal:
    driver: overlay
    attachable: false