ocrd-network: optional deploy mongodb and queue

ocrd_network/ocrd_network/deployer.py

@@ -17,7 +17,9 @@
 from .deployment_utils import (
     create_docker_client,
     DeployType,
-    wait_for_rabbitmq_availability
+    wait_for_rabbitmq_availability,
+    verify_mongodb_available,
+    verify_rabbitmq_available,
 )
 
 from .runtime_data import (
@@ -229,6 +231,23 @@ def deploy_rabbitmq(
             remove: bool,
             ports_mapping: Union[Dict, None] = None
     ) -> str:
+        if self.data_queue.skip_deployment:
+            self.log.debug(f"RabbitMQ is externaly managed. Skipping deployment")
+            verify_rabbitmq_available(
+                self.data_queue.address,
+                self.data_queue.port,
+                self.data_queue.vhost,
+                self.data_queue.username,
+                self.data_queue.password
+            )
+            wait_for_rabbitmq_availability(
+                host=self.data_queue.address,
+                port=int(self.data_queue.port),
+                vhost='/',
+                username=self.data_queue.username,
+                password=self.data_queue.password
+            )
+            return self.data_queue.url
         self.log.debug(f"Trying to deploy '{image}', with modes: "
                        f"detach='{detach}', remove='{remove}'")
 
@@ -289,6 +308,11 @@ def deploy_mongodb(
             remove: bool,
             ports_mapping: Union[Dict, None] = None
     ) -> str:
+        if self.data_mongo.skip_deployment:
+            self.log.debug('MongoDb is externaly managed. Skipping deployment')
+            verify_mongodb_available(self.data_mongo.url);
+            return self.data_mongo.url
+
         self.log.debug(f"Trying to deploy '{image}', with modes: "
                        f"detach='{detach}', remove='{remove}'")
 
@@ -305,11 +329,20 @@ def deploy_mongodb(
             ports_mapping = {
                 27017: self.data_mongo.port
             }
+        if self.data_mongo.username:
+            environment = [
+                f'MONGO_INITDB_ROOT_USERNAME={self.data_mongo.username}',
+                f'MONGO_INITDB_ROOT_PASSWORD={self.data_mongo.password}'
+            ]
+        else:
+            environment = []
+
         res = client.containers.run(
             image=image,
             detach=detach,
             remove=remove,
-            ports=ports_mapping
+            ports=ports_mapping,
+            environment=environment
         )
         if not res or not res.id:
             raise RuntimeError('Failed to start MongoDB docker container on host: '
@@ -322,7 +355,9 @@ def deploy_mongodb(
         return self.data_mongo.url
 
     def kill_rabbitmq(self) -> None:
-        if not self.data_queue.pid:
+        if self.data_queue.skip_deployment:
+            return
+        elif not self.data_queue.pid:
             self.log.warning('No running RabbitMQ instance found')
             return
         client = create_docker_client(
@@ -337,7 +372,9 @@ def kill_rabbitmq(self) -> None:
         self.log.info('The RabbitMQ is stopped')
 
     def kill_mongodb(self) -> None:
-        if not self.data_mongo.pid:
+        if self.data_mongo.skip_deployment:
+            return
+        elif not self.data_mongo.pid:
             self.log.warning('No running MongoDB instance found')
             return
         client = create_docker_client(

ocrd_network/ocrd_network/deployment_utils.py

@@ -4,8 +4,10 @@
 from docker.transport import SSHHTTPAdapter
 from paramiko import AutoAddPolicy, SSHClient
 from time import sleep
+import re
 
 from .rabbitmq_utils import RMQPublisher
+from pymongo import MongoClient
 
 __all__ = [
     'create_docker_client',
@@ -104,6 +106,24 @@ def wait_for_rabbitmq_availability(
     raise RuntimeError('Error waiting for queue startup: timeout exceeded')
 
 
+def verify_rabbitmq_available(host: str, port: int, vhost: str, username: str,
+                              password: str) -> None:
+    try:
+        dummy_publisher = RMQPublisher(host=host, port=port, vhost=vhost)
+        dummy_publisher.authenticate_and_connect(username=username, password=password)
+    except Exception:
+        raise Exception(f'Cannot connet to Rabbitmq host: {host}, port: {port}, '
+                        f'vhost: {vhost}, username: {username}')
+
+
+def verify_mongodb_available(mongo_url: str) -> None:
+    try:
+        client = MongoClient(mongo_url, serverSelectionTimeoutMS=1000.0)
+        client.admin.command("ismaster")
+    except Exception:
+        raise Exception(f'Cannot connet to MongoDb: {re.sub(r":[^@]+@", ":****@", mongo_url)}')
+
+
 class DeployType(Enum):
     """ Deploy-Type of the processing worker/processor server.
     """

ocrd_network/ocrd_network/runtime_data.py

@@ -98,12 +98,24 @@ class DataMongoDB:
     def __init__(self, config: Dict) -> None:
         self.address = config['address']
         self.port = int(config['port'])
-        self.ssh_username = config['ssh']['username']
-        self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
-        self.ssh_password = config['ssh'].get('password', None)
-        self.username = config['credentials']['username']
-        self.password = config['credentials']['password']
-        self.url = f'mongodb://{self.address}:{self.port}'
+        if 'ssh' in config:
+            self.ssh_username = config['ssh']['username']
+            self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
+            self.ssh_password = config['ssh'].get('password', None)
+        else:
+            self.ssh_username = None
+            self.ssh_keypath = None
+            self.ssh_password = None
+
+        if 'credentials' in config:
+            self.username = config['credentials']['username']
+            self.password = config['credentials']['password']
+            self.url = f'mongodb://{self.username}:{self.password}@{self.address}:{self.port}'
+        else:
+            self.username = None
+            self.password = None
+            self.url = f'mongodb://{self.address}:{self.port}'
+        self.skip_deployment = config.get('skip_deployment', False)
         # Assigned when deployed
         self.pid = None
 
@@ -112,12 +124,19 @@ class DataRabbitMQ:
     def __init__(self, config: Dict) -> None:
         self.address = config['address']
         self.port = int(config['port'])
-        self.ssh_username = config['ssh']['username']
-        self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
-        self.ssh_password = config['ssh'].get('password', None)
+        if 'ssh' in config:
+            self.ssh_username = config['ssh']['username']
+            self.ssh_keypath = config['ssh'].get('path_to_privkey', None)
+            self.ssh_password = config['ssh'].get('password', None)
+        else:
+            self.ssh_username = None
+            self.ssh_keypath = None
+            self.ssh_password = None
+
         self.vhost = '/'
         self.username = config['credentials']['username']
         self.password = config['credentials']['password']
         self.url = f'amqp://{self.username}:{self.password}@{self.address}:{self.port}{self.vhost}'
+        self.skip_deployment = config.get('skip_deployment', False)
         # Assigned when deployed
         self.pid = None

ocrd_validators/ocrd_validators/processing_server_config.schema.yml

@@ -26,6 +26,9 @@ properties:
       ssh:
         description: Information required for an SSH connection
         $ref: "#/$defs/ssh"
+      skip_deployment:
+        description: set to true to deploy queue yourself
+        type: boolean
   database:
     description: Information about the MongoDB
     type: object
@@ -46,6 +49,9 @@ properties:
       ssh:
         description: Information required for an SSH connection
         $ref: "#/$defs/ssh"
+      skip_deployment:
+        description: set to true to deploy database yourself
+        type: boolean
   hosts:
     description: A list of hosts where Processing Servers will be deployed
     type: array