docker-compose.yml

x-ingestor-common: &ingestor-common
  build:
    context: .
    dockerfile: services/ingestor/Dockerfile
    args:
      BUILDKIT_INLINE_CACHE: "1"
  depends_on:
    kafka:
      condition: service_healthy
    kafka-init:
      condition: service_completed_successfully
  networks:
    - pmon

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.6.1
    restart: unless-stopped
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - '2181:2181'
    networks:
      - pmon

  kafka:
    image: confluentinc/cp-kafka:7.6.1
    restart: unless-stopped
    depends_on:
      zookeeper:
        condition: service_started
    ports:
      - '9094:9094'
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:9094
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9094
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      # Auto-create topics when accessed (for development)
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: true
      KAFKA_DELETE_TOPIC_ENABLE: true
      # Cloud native optimizations
      KAFKA_SOCKET_REQUEST_MAX_BYTES: 104857600
      KAFKA_MESSAGE_MAX_BYTES: 1048576
      KAFKA_REPLICA_SOCKET_TIMEOUT_MS: 30000
      KAFKA_CONTROLLER_SOCKET_TIMEOUT_MS: 30000
      KAFKA_REQUEST_TIMEOUT_MS: 30000
      # Faster failure detection
      KAFKA_CONNECTIONS_MAX_IDLE_MS: 600000
      KAFKA_METADATA_MAX_AGE_MS: 300000
      # Better throughput
      KAFKA_NUM_NETWORK_THREADS: 3
      KAFKA_NUM_IO_THREADS: 8
      KAFKA_SOCKET_SEND_BUFFER_BYTES: 102400
      KAFKA_SOCKET_RECEIVE_BUFFER_BYTES: 102400
    networks:
      - pmon
    healthcheck:
      test: ["CMD-SHELL", "kafka-topics --bootstrap-server localhost:9092 --list"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 40s

  kafka-init:
    image: confluentinc/cp-kafka:7.6.1
    depends_on:
      kafka:
        condition: service_healthy
    networks:
      - pmon
    volumes:
      - ./scripts/kafka-init-topics.sh:/scripts/kafka-init-topics.sh:ro
    environment:
      KAFKA_BROKER: kafka:9092
      KAFKA_PARTITIONS: 3
      KAFKA_REPLICATION_FACTOR: 1
    command: ["/scripts/kafka-init-topics.sh"]
    restart: "no"

  redis:
    image: redis:7
    restart: unless-stopped
    ports:
      - '6379:6379'
    networks:
      - pmon

  ingestor-prices:
    <<: *ingestor-common
    environment:
      LOG_LEVEL: debug
      KAFKA_BROKER: kafka:9092
      INGEST_MANIFEST_PATH: /app/configs/ingest.yaml
      INGEST_SHARD_ID: hl-prices
      # Cloud native Kafka optimizations
      KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000
      KAFKA_PRODUCER_RETRIES: 3
      KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100
      KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000
      KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5
      KAFKA_PRODUCER_IDEMPOTENT: true
      # Backpressure and overload protection
      INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100
      INGEST_MAX_PENDING_PUBLISHES: 2000
      # Hyperliquid subscription rate limiting
      HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 2000
      HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4001/healthz"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped

  ingestor-orderbook:
    <<: *ingestor-common
    environment:
      LOG_LEVEL: debug
      KAFKA_BROKER: kafka:9092
      INGEST_MANIFEST_PATH: /app/configs/ingest.yaml
      INGEST_SHARD_ID: hl-orderbook
      # Cloud native Kafka optimizations
      KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000
      KAFKA_PRODUCER_RETRIES: 3
      KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100
      KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000
      KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5
      KAFKA_PRODUCER_IDEMPOTENT: true
      # Backpressure and overload protection
      INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100
      INGEST_MAX_PENDING_PUBLISHES: 2000
      # Hyperliquid subscription rate limiting
      HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 2500
      HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4001/healthz"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped

  ingestor-asset-metadata:
    <<: *ingestor-common
    environment:
      LOG_LEVEL: debug
      KAFKA_BROKER: kafka:9092
      INGEST_MANIFEST_PATH: /app/configs/ingest.yaml
      INGEST_SHARD_ID: hl-asset-metadata
      # Cloud native Kafka optimizations
      KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000
      KAFKA_PRODUCER_RETRIES: 3
      KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100
      KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000
      KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5
      KAFKA_PRODUCER_IDEMPOTENT: true
      # Backpressure and overload protection
      INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100
      INGEST_MAX_PENDING_PUBLISHES: 2000
      # Hyperliquid subscription rate limiting
      HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 3000
      HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4001/healthz"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped

  ingestor-overview:
    <<: *ingestor-common
    environment:
      LOG_LEVEL: debug
      KAFKA_BROKER: kafka:9092
      INGEST_MANIFEST_PATH: /app/configs/ingest.yaml
      INGEST_SHARD_ID: hl-overview
      # Cloud native Kafka optimizations
      KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000
      KAFKA_PRODUCER_RETRIES: 3
      KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100
      KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000
      KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5
      KAFKA_PRODUCER_IDEMPOTENT: true
      # Backpressure and overload protection
      INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100
      INGEST_MAX_PENDING_PUBLISHES: 2000
      # Hyperliquid subscription rate limiting
      HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 3500
      HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4001/healthz"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped

  ingestor-candles:
    <<: *ingestor-common
    environment:
      LOG_LEVEL: debug
      KAFKA_BROKER: kafka:9092
      INGEST_MANIFEST_PATH: /app/configs/ingest.yaml
      INGEST_SHARD_ID: hl-candles
      # Cloud native Kafka optimizations
      KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000
      KAFKA_PRODUCER_RETRIES: 3
      KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100
      KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000
      KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5
      KAFKA_PRODUCER_IDEMPOTENT: true
      # Backpressure and overload protection
      INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100
      INGEST_MAX_PENDING_PUBLISHES: 2000
      # Hyperliquid subscription rate limiting
      HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 4000
      HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4001/healthz"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped

  ingestor-trades:
    <<: *ingestor-common
    environment:
      LOG_LEVEL: debug
      KAFKA_BROKER: kafka:9092
      INGEST_MANIFEST_PATH: /app/configs/ingest.yaml
      INGEST_SHARD_ID: hl-trades
      # Cloud native Kafka optimizations
      KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000
      KAFKA_PRODUCER_RETRIES: 3
      KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100
      KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000
      KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5
      KAFKA_PRODUCER_IDEMPOTENT: true
      # Backpressure and overload protection
      INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100
      INGEST_MAX_PENDING_PUBLISHES: 2000
      # Hyperliquid subscription rate limiting
      HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 4500
      HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4001/healthz"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped

  ingestor-bbo:
    <<: *ingestor-common
    environment:
      LOG_LEVEL: debug
      KAFKA_BROKER: kafka:9092
      INGEST_MANIFEST_PATH: /app/configs/ingest.yaml
      INGEST_SHARD_ID: hl-bbo
      # Cloud native Kafka optimizations
      KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000
      KAFKA_PRODUCER_RETRIES: 3
      KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100
      KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000
      KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5
      KAFKA_PRODUCER_IDEMPOTENT: true
      # Backpressure and overload protection
      INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100
      INGEST_MAX_PENDING_PUBLISHES: 2000
      # Hyperliquid subscription rate limiting
      HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 5000
      HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4001/healthz"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped

  gateway:
    build:
      context: .
      dockerfile: services/gateway/Dockerfile
      args:
        BUILDKIT_INLINE_CACHE: "1"
    environment:
      - LOG_LEVEL=debug
      - PORT=4000
      - KAFKA_BROKER=kafka:9092
      - REDIS_URL=redis://redis:6379
      - INGEST_MANIFEST_PATH=/app/configs/ingest.yaml
      - STATS_REDIS_URL=redis://redis:6379
      - STATS_REDIS_NAMESPACE=pmon:stats:metrics:hyperliquid
      - GATEWAY_MEMORY_WARN_PCT=55
      - GATEWAY_MEMORY_CRITICAL_PCT=60
      - GATEWAY_MEMORY_FORCE_GC_PCT=55
      - GATEWAY_DEFAULT_WS_DOMAINS=prices,orderbook,metadata,overview,candles,trades,bbo
    ports:
      - '4000:4000'
    depends_on:
      kafka:
        condition: service_started
      redis:
        condition: service_started
      stats:
        condition: service_healthy
    networks:
      - pmon
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro

  stats:
    build:
      context: .
      dockerfile: services/stats/Dockerfile
      args:
        BUILDKIT_INLINE_CACHE: "1"
    environment:
      - LOG_LEVEL=debug
      - KAFKA_BROKER=kafka:9092
      - KAFKA_CLIENT_ID=pmon-stats-local
      - KAFKA_GROUP_ID=pmon-stats-local
      - REDIS_URL=redis://redis:6379
      - STATS_REDIS_NAMESPACE=pmon:stats:metrics
      - STATS_MANIFEST_PATH=/app/configs/ingest.yaml
      - STATS_PROVIDER_ID=hyperliquid
      - STATS_ADAPTER_ID=hyperliquid-perp
      - STATS_SERVER_PORT=4010
      - STATS_WINDOW_SIZE=30
    volumes:
      - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro
    ports:
      - '4010:4010'
    healthcheck:
      test:
        - CMD-SHELL
        - >
          node -e "globalThis.fetch('http://localhost:4010/healthz').then(res => { if (res.ok) { process.exit(0); } process.exit(1); }).catch(() => process.exit(1));"
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 60s
    depends_on:
      - kafka
      - redis
    networks:
      - pmon

  ui:
    build:
      context: .
      dockerfile: apps/ui/Dockerfile
      args:
        BUILDKIT_INLINE_CACHE: "1"
    environment:
      - GATEWAY_URL=http://gateway:4000
      - NEXT_PUBLIC_GATEWAY_WS_URL=ws://localhost:4000/ws
    ports:
      - '5173:5173'
    depends_on:
      - gateway
    networks:
      - pmon

  trader-intelligence:
    build:
      context: .
      dockerfile: services/trader-intelligence/Dockerfile
      args:
        BUILDKIT_INLINE_CACHE: "1"
    environment:
      NODE_ENV: production
      TRADER_INTELLIGENCE_PORT: "3004"
      TRADER_INTELLIGENCE_LOG_LEVEL: info
      REDIS_URL: redis://redis:6379
      REDIS_KEY_PREFIX: "intelligence:"
      HYPERLIQUID_API_URL: https://api.hyperliquid.xyz
      COLLECTION_BATCH_SIZE: "20"
      COLLECTION_CONCURRENCY: "5"
      KAFKA_ENABLED: "true"
      KAFKA_BROKERS: kafka:9092
      KAFKA_CLIENT_ID: trader-intelligence-local
      TRADER_INTEL_AUTO_SEED_ENABLED: "true"
      TRADER_INTEL_STATS_REDIS_URL: redis://redis:6379
      TRADER_INTEL_STATS_REDIS_NAMESPACE: pmon:stats:metrics:hyperliquid
      TRADER_INTEL_AUTO_SEED_ASSETS: "BTC,ETH,SOL,VNTL:SPACEX"
    ports:
      - '3004:3004'
    depends_on:
      redis:
        condition: service_started
      kafka:
        condition: service_healthy
      kafka-init:
        condition: service_completed_successfully
    networks:
      - pmon

networks:
  pmon:
    driver: bridge