Add RPC server support

README.rst

@@ -1,6 +1,7 @@
 ..
     SPDX-FileCopyrightText: 2019-2020 CERN.
     SPDX-FileCopyrightText: 2019-2020 Northwestern University.
+    SPDX-FileCopyrightText: 2025 Graz University of Technology.
     SPDX-License-Identifier: MIT
 
 =================
@@ -61,6 +62,58 @@ Local Development environment
     # Update assets or statics
     $ invenio-cli update
 
+Customizations
+==============
+
+It is possible to choose between two python package managers: `pipenv` and `uv`.
+`pipenv` is the default one. To customize the python package manager it is
+necessary to add the line `python_package_manager = VALUE` to the `.invenio`
+file. `VALUE` is `uv` or `pipenv`.
+
+It is possible to choose between two javascript package managers: `npm` and
+`pnpm`. `npm` is the default one. To customize the python package manager it is
+necessary to add the line `javascript_package_manager = VALUE` to the `.invenio`
+file. `VALUE` is `npm` or `pnpm`.
+
+It is possible to choose between two assets builders: `webpack` and `rspack`.
+`webpack` is the default one. To use `rspack` add `WEBPACKEXT_PROJECT =
+"invenio_assets.webpack:rspack_project"` to the `invenio.cfg` file.
+
+It is possible to send `invenio` commands to a long running RPC server instead
+of starting a new Python process for each command. To enable this add the line
+`use_rpc = true` to the `[cli]` section of the `.invenio` file. The server
+listens on a Unix domain socket at `<instance_path>/rpc.sock` and is started
+automatically when needed; invenio-cli stops it again on exit.
+
+The javascript package manager uses a lock file like the python package manager.
+This file `pnpm-lock.yaml` for `pnpm` and `packages-lock.json` for `npm` will be
+symlinked to the `var/instance/assets/` directory.
+
+Hints
+=====
+
+`uv`
+
+The development with `uv` is a little bit different than with `pipenv`. If there
+is a `uv.lock` file packages have to be updated manually or by removing the
+`uv.lock` file. The absence of the `uv.lock` file triggeres a new dependency
+resolving call which takes into account of new released packages. It would also
+be possible to use the `uv sync --upgrade` feature of `uv` but this installs
+also beta versions of packages which is not recommended. It may sound strange to
+remove the `uv.lock` file, but `uv` is that fast that deleting the `.venv`
+directory and the `uv.lock` file is the easiest and fastest and safest way to
+upgrade the packages.
+
+UV uses a local `.venv` directory. This will be created automatically, if not
+existing in the same directory as the `pyproject.toml` file.
+
+UV uses a `pyproject.toml` file.
+
+`rpc-server`
+
+To use `pnpm` with the `rpc-server` it is necessary to add
+`WEBPACKEXT_NPM_PKG_CLS = "pynpm:PNPMPackage"` to the `invenio.cfg` file.
+
 
 Containerized 'Production' environment
 --------------------------------------

invenio_cli/commands/assets.py

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2026 California Institute of Technology.
 # SPDX-FileCopyrightText: 2020 CERN.
+# SPDX-FileCopyrightText: 2025 Graz University of Technology.
 # SPDX-License-Identifier: MIT
 
 """Invenio module to ease the creation and management of applications."""

invenio_cli/commands/containers.py

@@ -137,7 +137,7 @@ def _setup(self, project_shortname="/opt/var/instance/"):
             ),
         ]
 
-        if rdm_version()[0] >= 10:
+        if rdm_version(self.cli_config)[0] >= 10:
             steps.extend(
                 [
                     FunctionStep(
@@ -159,11 +159,11 @@ def _setup(self, project_shortname="/opt/var/instance/"):
                 ]
             )
 
-        if rdm_version()[0] >= 11:
+        if rdm_version(self.cli_config)[0] >= 11:
             steps.extend(self.rdm_fixtures(project_shortname))
             steps.extend(self.translations(project_shortname))
 
-        if rdm_version()[0] >= 12:
+        if rdm_version(self.cli_config)[0] >= 12:
             steps.extend(self.declare_queues(project_shortname))
 
         return steps

invenio_cli/commands/install.py

@@ -5,7 +5,6 @@
 """Invenio module to ease the creation and management of applications."""
 
 from ..helpers import filesystem
-from ..helpers.process import run_cmd
 from .local import LocalCommands
 from .packages import PackagesCommands
 from .steps import FunctionStep
@@ -33,17 +32,16 @@ def install_py_dependencies(self, pre, dev=False):
 
     def update_instance_path(self):
         """Update path to instance in config."""
-        result = run_cmd(
-            self.cli_config.python_package_manager.run_command(
-                "invenio",
-                "shell",
-                # make sure the shell does not append cursor if editing_mode is set to `vi` in config
-                "--TerminalInteractiveShell.editing_mode=''",
-                "--no-term-title",
-                "-c",
-                "\"print(app.instance_path, end='')\"",
-            )
+        op = self.cli_config.python_package_manager.invenio_command(
+            "invenio",
+            "shell",
+            # make sure the shell does not append cursor if editing_mode is set to `vi` in config
+            "--TerminalInteractiveShell.editing_mode=''",
+            "--no-term-title",
+            "-c",
+            "print(app.instance_path, end='')",
         )
+        result = op(capture=True)
         if result.status_code == 0:
             self.cli_config.update_instance_path(result.output.strip())
             result.output = "Instance path updated successfully."

invenio_cli/commands/local.py

@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: 2026 California Institute of Technology.
 # SPDX-FileCopyrightText: 2020 CERN.
-# SPDX-FileCopyrightText: 2022 Graz University of Technology.
+# SPDX-FileCopyrightText: 2022-2026 Graz University of Technology.
 # SPDX-FileCopyrightText: 2026 Northwestern University.
 # SPDX-License-Identifier: MIT
 
@@ -17,7 +17,9 @@
 import click
 
 from ..helpers import env, filesystem
-from ..helpers.process import ProcessResponse, run_interactive
+from ..helpers.package_managers import LocalOp
+from ..helpers.process import ProcessResponse
+from ..helpers.rpc import RPCOp
 from ..helpers.versions import rdm_version
 from .commands import Commands
 
@@ -106,22 +108,21 @@ def update_statics_and_assets(self, force, debug=False, log_file=None):
         # Commands
         py_pkg_man = self.cli_config.python_package_manager
         js_pkg_man = self.cli_config.javascript_package_manager
-        ops = [py_pkg_man.run_command("invenio", "collect", "--verbose")]
+        ops = [py_pkg_man.invenio_command("invenio", "collect", "--verbose")]
 
         if force:
-            ops.append(py_pkg_man.run_command("invenio", "webpack", "clean", "create"))
+            ops.append(
+                py_pkg_man.invenio_command("invenio", "webpack", "clean", "create")
+            )
             # We need to copy the js lock files here, since webpack regenerates
             # package.json and we want the locked version instead.
             ops.append(self._copy_js_lock_files)
-            ops.append(py_pkg_man.run_command("invenio", "webpack", "install"))
+            ops.append(py_pkg_man.invenio_command("invenio", "webpack", "install"))
         else:
-            ops.append(py_pkg_man.run_command("invenio", "webpack", "create"))
-            # We need to copy the js lock files here, since webpack regenerates
-            # package.json and we want the locked version instead.
+            ops.append(py_pkg_man.invenio_command("invenio", "webpack", "create"))
             ops.append(self._copy_js_lock_files)
         ops.append(self._statics)
-        ops.append(py_pkg_man.run_command("invenio", "webpack", "build"))
-
+        ops.append(py_pkg_man.invenio_command("invenio", "webpack", "build"))
         # Keep the same messages for some of the operations for backward compatibility
         messages = {
             "build": "Building assets...",
@@ -130,16 +131,15 @@ def update_statics_and_assets(self, force, debug=False, log_file=None):
 
         with env(FLASK_DEBUG="1" if debug else "0"):
             for op in ops:
-                if callable(op):
-                    response = op()
-                else:
-                    if op[-1] in messages:
-                        click.secho(messages[op[-1]], fg="green")
-                    response = run_interactive(
-                        op,
+                if isinstance(op, (LocalOp, RPCOp)):
+                    if op.label in messages:
+                        click.secho(messages[op.label], fg="green")
+                    response = op(
                         env={"PIPENV_VERBOSITY": "-1", **js_pkg_man.env_overrides()},
                         log_file=log_file,
                     )
+                else:
+                    response = op()
                 if response.status_code != 0:
                     break
         return response
@@ -224,7 +224,7 @@ def run_worker(
     def run_jobs_scheduler(self, celery_log_file=None, celery_log_level="INFO"):
         """Run Celery beat scheduler for jobs."""
         # Jobs scheduler is only available in RDM v13+
-        version = rdm_version()
+        version = rdm_version(self.cli_config)
         if version is None:
             click.secho(
                 "RDM version couldn't be determined. Not running jobs scheduler.",

invenio_cli/commands/services.py

@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: 2020-2024 CERN.
 # SPDX-FileCopyrightText: 2021 Esteban J. G. Gabancho.
-# SPDX-FileCopyrightText: 2024 Graz University of Technology.
+# SPDX-FileCopyrightText: 2024-2025 Graz University of Technology.
 # SPDX-License-Identifier: MIT
 
 """Invenio module to ease the creation and management of applications."""
@@ -38,7 +38,12 @@ def ensure_containers_running(self):
             cmd_env = {"INSTANCE_PATH": str(instance_path)}
         # Set environment variable for the instance path, it might be needed by docker services
         with env(**cmd_env):
-            self.docker_helper.start_containers()
+            response = self.docker_helper.start_containers()
+        if response.status_code != 0:
+            return ProcessResponse(
+                error="Failed to start containers (see docker compose output above).",
+                status_code=1,
+            )
 
         services = ["redis", self.cli_config.get_db_type(), "search"]
         for service in services:
@@ -84,7 +89,7 @@ def _cleanup(self):
         pkg_man = self.cli_config.python_package_manager
         steps = [
             CommandStep(
-                cmd=pkg_man.run_command(
+                cmd=pkg_man.invenio_command(
                     "invenio",
                     "shell",
                     "--no-term-title",
@@ -96,13 +101,13 @@ def _cleanup(self):
                 skippable=True,
             ),
             CommandStep(
-                cmd=pkg_man.run_command("invenio", "db", "destroy", "--yes-i-know"),
+                cmd=pkg_man.invenio_command("invenio", "db", "destroy", "--yes-i-know"),
                 env={"PIPENV_VERBOSITY": "-1"},
                 message="Destroying database...",
                 skippable=True,
             ),
             CommandStep(
-                cmd=pkg_man.run_command(
+                cmd=pkg_man.invenio_command(
                     "invenio",
                     "index",
                     "destroy",
@@ -114,7 +119,9 @@ def _cleanup(self):
                 skippable=True,
             ),
             CommandStep(
-                cmd=pkg_man.run_command("invenio", "index", "queue", "init", "purge"),
+                cmd=pkg_man.invenio_command(
+                    "invenio", "index", "queue", "init", "purge"
+                ),
                 env={"PIPENV_VERBOSITY": "-1"},
                 message="Purging queues...",
                 skippable=True,
@@ -132,7 +139,7 @@ def _default_location_path(self):
         """Build default location path based on file storage selection."""
         file_storage = self.cli_config.get_file_storage()
         if file_storage == "local":
-            return "{}/data".format(self.cli_config.get_instance_path())
+            return "{}/data".format(self.cli_config.get_data_path())
         return "{}://default".format(self.cli_config.get_file_storage().lower())
 
     def _setup(self, demo_data=False):
@@ -145,12 +152,12 @@ def _setup(self, demo_data=False):
                 message="Checking services are not setup...",
             ),
             CommandStep(
-                cmd=pkg_man.run_command("invenio", "db", "init", "create"),
+                cmd=pkg_man.invenio_command("invenio", "db", "init", "create"),
                 env={"PIPENV_VERBOSITY": "-1"},
                 message="Creating database...",
             ),
             CommandStep(
-                cmd=pkg_man.run_command(
+                cmd=pkg_man.invenio_command(
                     "invenio",
                     "files",
                     "location",
@@ -163,12 +170,12 @@ def _setup(self, demo_data=False):
                 message="Creating files location...",
             ),
             CommandStep(
-                cmd=pkg_man.run_command("invenio", "roles", "create", "admin"),
+                cmd=pkg_man.invenio_command("invenio", "roles", "create", "admin"),
                 env={"PIPENV_VERBOSITY": "-1"},
                 message="Creating admin role...",
             ),
             CommandStep(
-                cmd=pkg_man.run_command(
+                cmd=pkg_man.invenio_command(
                     "invenio",
                     "access",
                     "allow",
@@ -180,19 +187,19 @@ def _setup(self, demo_data=False):
                 message="Allowing superuser access to admin role...",
             ),
             CommandStep(
-                cmd=pkg_man.run_command("invenio", "index", "init"),
+                cmd=pkg_man.invenio_command("invenio", "index", "init"),
                 env={"PIPENV_VERBOSITY": "-1"},
                 message="Creating indices...",
             ),
         ]
 
-        rdm_version_value = rdm_version()
+        rdm_version_value = rdm_version(self.cli_config)
         if rdm_version_value:
             if rdm_version_value[0] >= 10:
                 steps.extend(
                     [
                         CommandStep(
-                            cmd=pkg_man.run_command(
+                            cmd=pkg_man.invenio_command(
                                 "invenio",
                                 "rdm-records",
                                 "custom-fields",
@@ -202,7 +209,7 @@ def _setup(self, demo_data=False):
                             message="Creating custom fields for records...",
                         ),
                         CommandStep(
-                            cmd=pkg_man.run_command(
+                            cmd=pkg_man.invenio_command(
                                 "invenio",
                                 "communities",
                                 "custom-fields",
@@ -232,9 +239,10 @@ def _setup(self, demo_data=False):
             )
 
         if ils_version():
-            cmd = pkg_man.run_command("invenio", "setup", "--verbose")
+            args = ["invenio", "setup", "--verbose"]
             if not demo_data:
-                cmd.append("--skip-demo-data")
+                args.append("--skip-demo-data")
+            cmd = pkg_man.invenio_command(*args)
             steps.extend(
                 [
                     CommandStep(
@@ -260,7 +268,7 @@ def demo(self):
         pkg_man = self.cli_config.python_package_manager
         steps = [
             CommandStep(
-                cmd=pkg_man.run_command("invenio", "rdm-records", "demo"),
+                cmd=pkg_man.invenio_command("invenio", "rdm-records", "demo"),
                 env={"PIPENV_VERBOSITY": "-1"},
                 message="Creating demo records...",
             )
@@ -271,14 +279,14 @@ def demo(self):
     def declare_queues(self):
         """Steps to declare the MQ queues required for statistics, etc."""
         pkg_man = self.cli_config.python_package_manager
-        command = pkg_man.run_command("invenio", "queues", "declare")
+        command = pkg_man.invenio_command("invenio", "queues", "declare")
         steps = [CommandStep(cmd=command, message="Declaring queues...")]
         return steps
 
     def fixtures(self):
         """Steps to set up the required fixtures for the instance."""
         pkg_man = self.cli_config.python_package_manager
-        command = pkg_man.run_command("invenio", "rdm-records", "fixtures")
+        command = pkg_man.invenio_command("invenio", "rdm-records", "fixtures")
         steps = [
             CommandStep(
                 cmd=command,
@@ -292,7 +300,7 @@ def fixtures(self):
     def rdm_fixtures(self):
         """Steps to set up the rdm fixtures for the instance."""
         pkg_man = self.cli_config.python_package_manager
-        command = pkg_man.run_command("invenio", "rdm", "fixtures")
+        command = pkg_man.invenio_command("invenio", "rdm", "fixtures")
         steps = [
             CommandStep(
                 cmd=command,