docker-compose.yml

x-logging: &default-logging
  driver: 'json-file'
  options:
    max-size: '10m'
    max-file: '3'
x-virtuoso-healthcheck: &virtuoso-healthcheck
  # Virtuoso is healthy when its SPARQL endpoint answers a query.
  test:
    - CMD-SHELL
    - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8890/sparql?query=ASK%20%7B%3Fs%20%3Fp%20%3Fo%7D')"
  interval: 10s
  timeout: 5s
  retries: 10
  start_period: 5m
x-database-healthcheck: &database-healthcheck
  # The database (mu-authorization / sparql-parser) is healthy when it
  # answers an (anonymous, public) ASK query.
  test:
    - CMD-SHELL
    - wget -q -O /dev/null 'http://localhost:8890/sparql?query=ASK%20%7B%3Fs%20%3Fp%20%3Fo%7D'
  interval: 10s
  timeout: 5s
  retries: 5
  start_period: 30s
x-depends-on-virtuoso: &depends-on-virtuoso
  virtuoso:
    condition: service_healthy
x-depends-on-database: &depends-on-database
  database:
    condition: service_healthy
x-deltanotifier-healthcheck: &deltanotifier-healthcheck
  test:
    - CMD-SHELL
    - node -e "require('http').get('http://localhost:80/', () => process.exit(0)).on('error', () => process.exit(1))"
  interval: 10s
  timeout: 5s
  retries: 5
  start_period: 30s
x-depends-on-database-and-delta: &depends-on-database-and-delta
  database:
    condition: service_healthy
  deltanotifier:
    condition: service_healthy
services:
  identifier:
    image: semtech/mu-identifier:1.11.0
    environment:
      DEFAULT_MU_AUTH_ALLOWED_GROUPS_HEADER: '[{"variables":[],"name":"public"}]'
      DEFAULT_ACCESS_CONTROL_ALLOW_ORIGIN_HEADER: '*'
      SESSION_COOKIE_SECURE: 'true'
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  dispatcher:
    image: semtech/mu-dispatcher:2.1.1
    volumes:
      - ./config/dispatcher:/config
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  database:
    image: semtech/sparql-parser:0.0.15
    environment:
      LISP_DYNAMIC_SPACE_SIZE: 8192
    volumes:
      - ./config/authorization:/config
      - ./data/authorization:/data
    depends_on: *depends-on-virtuoso
    healthcheck: *database-healthcheck
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  virtuoso:
    image: redpencil/virtuoso:1.4.0
    environment:
      SPARQL_UPDATE: 'true'
    volumes:
      - ./config/virtuoso/virtuoso.ini:/data/virtuoso.ini
      - ./data/db:/data
    healthcheck: *virtuoso-healthcheck
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
    ports:
      - 8890:8890
  resource:
    image: semtech/mu-cl-resources:1.27.2
    environment:
      CACHE_CLEAR_PATH: 'http://cache/.mu/clear-keys'
    volumes:
      - ./config/resources:/config
    depends_on: *depends-on-database-and-delta
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  cache:
    image: semtech/mu-cache:2.0.1
    links:
      - resource:backend
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  sparql-cache:
    image: redpencil/varnish-post:1.0.0
    environment:
      BACKEND_HOST: database
      CACHE_TTL: 3600s
    volumes:
      - ./config/varnish/default.vcl:/templates/default.vcl:ro
    depends_on: *depends-on-database
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  file:
    image: semtech/mu-file-service:3.4.0
    volumes:
      - ./data/files:/share
    depends_on: *depends-on-database
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
    environment:
      MU_APPLICATION_GRAPH: "http://mu.semte.ch/graphs/harvesting"
  project-scripts:
    image: semtech/simple-script-store:1.0.0
    volumes:
      - ./scripts/:/app/scripts/
    restart: 'no'
  migrations:
    image: semtech/mu-migrations-service:0.9.0
    environment:
      MU_SPARQL_ENDPOINT: http://virtuoso:8890/sparql
    depends_on: *depends-on-virtuoso
    volumes:
      - ./config/migrations:/data/migrations
  deltanotifier:
    image: nvdk/mu-delta-notifier:1.1.0
    volumes:
      - ./config/delta:/config
    healthcheck: *deltanotifier-healthcheck
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  frontend-yasgui:
    image: lblod/frontend-decide-yasgui:0.0.4
    environment:
      # The following are environment-specific and should be set in the
      # environment's appropriate override file.
      EMBER_OAUTH_API_KEY: '<OAUTH_API_KEY>'
      EMBER_OAUTH_API_REDIRECT_URL: '<OAUTH_API_REDIRECT_URL>'
      EMBER_OAUTH_API_BASE_URL: '<OAUTH_API_BASE_URL>'
      EMBER_OAUTH_API_LOGOUT_URL: '<OAUTH_API_LOGOUT_URL'
      EMBER_OAUTH_API_SCOPE: '<OAUTH_API_SCOPE>'
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  frontend-harvesting:
    image: lblod/frontend-harvesting-self-service:feature-oparl-harvesting
    environment:
      EMBER_AUTHENTICATION_ENABLED: 'true'
      EMBER_LOGIN_ROUTE: 'login'
      EMBER_OPARL_HARVESTING_ENABLED: false
      EMBER_PDF_HARVESTING_ENABLED: false
      EMBER_NER_AND_NEL_ANNOTATIONS_ENABLED: true
      EMBER_CODELIST_MAPPING_ENABLED: true
      EMBER_OSLO_HARVESTING_ENABLED: true
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  login:
    image: semtech/mu-login-service:3.0.0
    links:
      - database:database
    environment:
      USERS_GRAPH: 'http://mu.semte.ch/graphs/users'
      SESSIONS_GRAPH: 'http://mu.semte.ch/graphs/sessions'
    depends_on: *depends-on-database
    restart: always
    logging: *default-logging
    labels:
      - 'logging=true'
  job-controller:
    image: lblod/job-controller-service:1.4.1
    volumes:
      - ./config/job-controller:/config
    environment:
      MU_APPLICATION_GRAPH: 'http://mu.semte.ch/graphs/harvesting'
      LOG_SPARQL_ALL: 'false'
      DEBUG_AUTH_HEADERS: 'false'
    depends_on: *depends-on-database-and-delta
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  scheduled-job-controller:
    image: lblod/scheduled-job-controller-service:1.3.0
    environment:
      LOG_SPARQL_ALL: 'false'
      DEBUG_AUTH_HEADERS: 'false'
      MAX_CONCURRENT_JOBS: 3
      MU_APPLICATION_GRAPH: 'http://mu.semte.ch/graphs/harvesting'
    depends_on: *depends-on-database-and-delta
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  annotation-job-splitter:
    image: lblod/annotation-job-splitter-service:0.0.6
    volumes:
      - ./config/annotation-job-splitter:/config
    depends_on: *depends-on-database-and-delta
    restart: always
    logging: *default-logging
    labels:
      - 'logging=true'
  oslo-eli-transformer:
    image: lblod/decide-harvester-transformation-service:1.1.0
    environment:
      INPUT_GRAPH: http://mu.semte.ch/graphs/public
      OUTPUT_GRAPH: http://mu.semte.ch/graphs/public/gent
      ORGANIZATIONS_GRAPH: http://mu.semte.ch/graphs/bestuurseenheden-bestuursorganen
      BYPASS_MU_SPARQL_ENDPOINT: true
      BATCH_SIZE: 1000
      SLEEP_BETWEEN_BATCHES: 5
      OPERATION_URI: http://lblod.data.gift/id/jobs/concept/TaskOperation/oslo-eli/transform
      DROP_INPUT_GRAPH: false
    depends_on: *depends-on-database-and-delta
    labels:
      - logging=true
    restart: always
    logging: *default-logging
  ##################################
  ### elastic and embeddings
  ##################################
  search:
    image: semtech/mu-search:feature-large-dense-vector
    environment:
      INCLUDE_SCORE: true
    volumes:
      - ./config/search:/config
  elasticsearch:
    image: semtech/mu-search-elastic-backend:feature-bump-es-9.2.0
    volumes:
      - ./data/elasticsearch/:/usr/share/elasticsearch/data
    environment:
      - discovery.type=single-node
  embedding:
    image: semanticai/embedding-service:0.0.4
    labels:
      - logging=true
    environment:
      TZ: Europe/Brussels
      OLLAMA_HOST: 'http://ollama:11434'
      LOG_SPARQL_ALL: 'false'
    volumes:
      - ./config/embedding/config.py:/app/config.py
  ##############################################################################
  # AI PIPELINES
  ##############################################################################
  entity-linking:
    image: semanticai/entity-linking-backend:0.2.4
    environment:
      TARGET_GRAPH: 'http://mu.semte.ch/graphs/public'
      DEFAULT_GRAPH: 'http://mu.semte.ch/graphs/harvesting'
      JOB_GRAPH: 'http://mu.semte.ch/graphs/harvesting'
      AI_GRAPH: 'http://mu.semte.ch/graphs/public/ai'
      ALLOW_MU_AUTH_SUDO: 'true'
      OLLAMA_HOST: 'http://ollama:11434'
      NOMINATIM_ENDPOINT: 'http://nominatim:8080/'
      MCP_SERVER_URL: 'http://localhost:80/mcp/sse'
      ENABLED_TOOLS: 'search_sparql_docs,execute_sparql_query,search_location'
      LLM_PROVIDER: 'mistral'
      LLM_MAX_RETRIES: 3
      MISTRAL_MODEL: 'ministral-14b-2512'
      # 0.2.x moved the config mount from /app to /config
      CONFIG_PATH: '/config/config.json'
      # decide_ai_service_base reads this compose file at startup to record the
      # running image tag/config as agent provenance in the store. It looks up
      # services[COMPOSE_SERVICE], so this must match the service name below and
      # the mounted file must contain it.
      COMPOSE_FILE: /tmp/docker-compose.yaml
      COMPOSE_SERVICE: entity-linking
      MISTRAL_API_KEY: ${MISTRAL_API_KEY}
    links:
      - database:database
    volumes:
      - ./config/nel/config:/config
      - ./data/nel:/app/data
      - ./docker-compose.yml:/tmp/docker-compose.yaml:ro
    depends_on: *depends-on-database-and-delta
    restart: always
  area-classification:
    image: redpencil/decide-area-classification-service:0.1.0
    environment:
      OPERATION_URI: 'http://lblod.data.gift/id/jobs/concept/TaskOperation/area-classifying'
      AI_GRAPH: 'http://mu.semte.ch/graphs/public/ai'
      WIJK_GRAPH: 'http://mu.semte.ch/graphs/public'
      # The geo query uses bif: functions, which sparql-parser will not pass
      # through, so reads go straight to Virtuoso. Writes still go via mu-auth
      # so deltas keep firing.
      GEO_SPARQL_ENDPOINT: 'http://virtuoso:8890/sparql'
      CONFIDENCE_THRESHOLD: '0.9'
      ALLOW_MU_AUTH_SUDO: 'true'
    volumes:
      - ./config/area-classification:/config
    links:
      - database:database
    depends_on: *depends-on-database-and-delta
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging
  pdf-scraper:
    image: semanticai/decide-pdf-scraper:0.0.6
    environment:
      TARGET_GRAPH: http://mu.semte.ch/graphs/harvesting
      JOB_GRAPH: http://mu.semte.ch/graphs/harvesting
      DATA_CONTAINER_GRAPH: http://mu.semte.ch/graphs/harvesting
      HARVEST_COLLECTIONS_GRAPH: http://mu.semte.ch/graphs/harvesting
      REMOTE_OBJECT_GRAPH: http://mu.semte.ch/graphs/harvesting
      FILES_GRAPH: http://mu.semte.ch/graphs/harvesting
      OPARL_TEMP_GRAPH: http://mu.semte.ch/graphs/harvesting
      OSLO_TEMP_GRAPH: http://mu.semte.ch/graphs/harvesting

      PUBLICATION_GRAPH: http://mu.semte.ch/graphs/public/pdf
      EXPRESSIONS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      WORKS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      MANIFESTATIONS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      AI_GRAPH: http://mu.semte.ch/graphs/public/pdf
      ALLOW_MU_AUTH_SUDO: true
    labels:
      - logging=true
    logging: *default-logging
    restart: none
    entrypoint: ["echo", "Service pdf-scraper is disabled"]
  pdf-content:
    image: semanticai/decide-pdf-content-extraction:0.2.3
    environment:
      APACHE_TIKA_URL: 'http://apache-tika:9998/tika'
      TARGET_GRAPH: http://mu.semte.ch/graphs/harvesting
      JOB_GRAPH: http://mu.semte.ch/graphs/harvesting
      DATA_CONTAINER_GRAPH: http://mu.semte.ch/graphs/harvesting
      HARVEST_COLLECTIONS_GRAPH: http://mu.semte.ch/graphs/harvesting
      REMOTE_OBJECT_GRAPH: http://mu.semte.ch/graphs/harvesting
      FILES_GRAPH: http://mu.semte.ch/graphs/harvesting
      OPARL_TEMP_GRAPH: http://mu.semte.ch/graphs/harvesting
      OSLO_TEMP_GRAPH: http://mu.semte.ch/graphs/harvesting

      PUBLICATION_GRAPH: http://mu.semte.ch/graphs/public/pdf
      EXPRESSIONS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      WORKS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      MANIFESTATIONS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      AI_GRAPH: http://mu.semte.ch/graphs/public/pdf
      ALLOW_MU_AUTH_SUDO: true
      MOUNTED_SHARE_FOLDER: '/mnt/share'
    volumes:
      - ./data/files:/mnt/share
      - ./config/pdf-content/config.json:/app/config.json:ro # Mount config as read-only
    labels:
      - logging=true
    restart: none
    logging: *default-logging
    entrypoint: ["echo", "Service pdf-content is disabled"]
  apache-tika:
    image: lblod/apache-tika-service:0.0.3
    restart: none
    logging: *default-logging
    entrypoint: ["echo", "Service apache-tika is disabled"]
  nominatim:
    image: mediagis/nominatim:5.1
    environment:
      - PBF_URL=https://download.geofabrik.de/europe/belgium-latest.osm.pbf
    restart: always
    volumes:
      - ./data/nominatim:/var/lib/postgresql/16/main
  named-entity-recognition:
    image: semanticai/decide-geocoding-service:0.2.4
    environment:
      TARGET_GRAPH: http://mu.semte.ch/graphs/harvesting
      JOB_GRAPH: http://mu.semte.ch/graphs/harvesting
      DATA_CONTAINER_GRAPH: http://mu.semte.ch/graphs/harvesting
      HARVEST_COLLECTIONS_GRAPH: http://mu.semte.ch/graphs/harvesting
      REMOTE_OBJECT_GRAPH: http://mu.semte.ch/graphs/harvesting
      FILES_GRAPH: http://mu.semte.ch/graphs/harvesting
      OPARL_TEMP_GRAPH: http://mu.semte.ch/graphs/harvesting
      OSLO_TEMP_GRAPH: http://mu.semte.ch/graphs/harvesting
      PUBLICATION_GRAPH: http://mu.semte.ch/graphs/public/pdf
      EXPRESSIONS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      WORKS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      MANIFESTATIONS_GRAPH: http://mu.semte.ch/graphs/public/pdf
      AI_GRAPH: http://mu.semte.ch/graphs/public/ai
      ALLOW_MU_AUTH_SUDO: true
      # 0.2.x moved the config mount from /app to /config
      CONFIG_PATH: /config/config.json
      # See entity-linking: read at startup to record agent provenance.
      COMPOSE_FILE: /tmp/docker-compose.yaml
      COMPOSE_SERVICE: named-entity-recognition
      MISTRAL_API_KEY: ${MISTRAL_API_KEY}
    volumes:
      - ./config/ner/config.json:/config/config.json:ro # Mount config as read-only
      - ./docker-compose.yml:/tmp/docker-compose.yaml:ro
    depends_on: *depends-on-virtuoso
    labels:
      - logging=true
    restart: always
    logging: *default-logging
  harvest_singleton-job:
    image: lblod/harvesting-singleton-job-service:1.0.0
    depends_on: *depends-on-database-and-delta
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging
  harvest_scraper:
    image: lblod/scraper:1.6.0
    volumes:
      - ./data/files:/share
    environment:
      DEFAULT_GRAPH: "http://mu.semte.ch/graphs/harvesting"
      INCREMENTAL_RETRIEVAL: "true"
      STORE_ALL_PAGES: "false"
    depends_on: *depends-on-database-and-delta
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging
  # harvest_compression:
  #   image: lblod/job-compression-service:0.2.4
  #   volumes:
  #     - ./data/files:/share
  #   environment:
  #     DEFAULT_GRAPH: "http://mu.semte.ch/graphs/harvesting"
  #     HIGH_LOAD_DATABASE_ENDPOINT: "http://virtuoso:8890/sparql"
  #   labels:
  #     - "logging=true"
  #   restart: always
  #   logging: *default-logging
  harvest_extract:
    image: lblod/harvesting-extract-to-ttl:0.15.0
    environment:
      VIRTUOSO_ENDPOINT: "http://virtuoso:8890/sparql"
      SUDO_QUERY_RETRY: "true"
      TARGET_GRAPH: "http://mu.semte.ch/graphs/harvesting"
      NODE_OPTIONS: "--max-old-space-size=4096" # needed with the new rdfa parser
      WRITE_DEBUG_TTLS: "false"
    volumes:
      - ./data/files:/share
    depends_on: *depends-on-database-and-delta
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging

  harvest_cleanup:
    image: lblod/local-harvesting-cleanup-previous-jobs-service
    environment:
      MAX_DAYS_TO_KEEP_SUCCESSFUL_JOBS: 30
      MAX_DAYS_TO_KEEP_BUSY_JOBS: 7
      MAX_DAYS_TO_KEEP_FAILED_JOBS: 7
      LOG_SPARQL_ALL: "false"
      DEBUG_AUTH_HEADERS: "false"
    volumes:
      - ./data/files:/share
    depends_on: *depends-on-database-and-delta
    restart: always
    logging: *default-logging

  harvest_validate:
    image: lblod/harvesting-validator:0.4.0
    environment:
      # SLEEP_BETWEEN_JOB_MS: 100 --> replaced by a semaphore (max nbcpu conccurent tasks)
      TARGET_GRAPH: "http://mu.semte.ch/graphs/harvesting"
      STRICT_MODE_FILTERING: "false"
      LIMIT_SIZE: 2000
    volumes:
      - ./data/files:/share
      - ./config/shacl:/config
    depends_on: *depends-on-database-and-delta
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging

  harvest_diff:
    image: lblod/harvesting-diff-service:0.3.0
    environment:
      # SLEEP_BETWEEN_JOB_MS: 100 --> replaced by a semaphore (max nbcpu conccurent tasks)
      TARGET_GRAPH: "http://mu.semte.ch/graphs/harvesting"
      PUBLICATION_GRAPH: "http://mu.semte.ch/graphs/public"
      LIMIT_SIZE: 2000
    volumes:
      - ./data/files:/share
    depends_on: *depends-on-database-and-delta
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging

  harvest_sameas:
    image: lblod/import-with-sameas-service:4.9.0
    environment:
      RENAME_DOMAIN: "http://data.lblod.info/id/"
      TARGET_GRAPH: "http://mu.semte.ch/graphs/public"
      HIGH_LOAD_DATABASE_ENDPOINT: "http://virtuoso:8890/sparql"
      ROLLBACK_ENABLED: "false"
      TASK_TIMEOUT_HOURS: "20"
      LOG_SPARQL_ALL: "false"
      DEBUG_AUTH_HEADERS: "false"
    volumes:
      - ./config/same-as-service:/config
      - ./data/files:/share
    depends_on: *depends-on-database-and-delta
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging
  harvest_gen_delta:
    image: lblod/harvesting-generation-delta-service:0.1.1
    environment:
      TARGET_DIRECTORY_DELTA_PATH: "/share/deltas/besluiten"
      BUFFER_SIZE: "100"
      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager-besluiten"
      TARGET_PUBLISHER_GRAPH: "http://redpencil.data.gift/id/deltas/producer/lblod-harvester-besluiten-producer"
      HIGH_LOAD_DATABASE_ENDPOINT: "http://virtuoso:8890/sparql"
    volumes:
      - ./data/files:/share
    depends_on: *depends-on-database-and-delta
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging

  metrics:
    image: lblod/metrics-service:0.1.0
    volumes:
      - ./data/metrics:/external-metrics
      - ./config/metrics:/config
    labels:
      - 'logging=true'
    restart: always
    logging: *default-logging