diff --git a/README.rst b/README.rst index 65be280..aa45e72 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,7 @@ .. SPDX-FileCopyrightText: 2019-2020 CERN. SPDX-FileCopyrightText: 2019-2020 Northwestern University. + SPDX-FileCopyrightText: 2025 Graz University of Technology. SPDX-License-Identifier: MIT ================= @@ -61,6 +62,58 @@ Local Development environment # Update assets or statics $ invenio-cli update +Customizations +============== + +It is possible to choose between two python package managers: `pipenv` and `uv`. +`pipenv` is the default one. To customize the python package manager it is +necessary to add the line `python_package_manager = VALUE` to the `.invenio` +file. `VALUE` is `uv` or `pipenv`. + +It is possible to choose between two javascript package managers: `npm` and +`pnpm`. `npm` is the default one. To customize the python package manager it is +necessary to add the line `javascript_package_manager = VALUE` to the `.invenio` +file. `VALUE` is `npm` or `pnpm`. + +It is possible to choose between two assets builders: `webpack` and `rspack`. +`webpack` is the default one. To use `rspack` add `WEBPACKEXT_PROJECT = +"invenio_assets.webpack:rspack_project"` to the `invenio.cfg` file. + +It is possible to send `invenio` commands to a long running RPC server instead +of starting a new Python process for each command. To enable this add the line +`use_rpc = true` to the `[cli]` section of the `.invenio` file. The server +listens on a Unix domain socket at `/rpc.sock` and is started +automatically when needed; invenio-cli stops it again on exit. + +The javascript package manager uses a lock file like the python package manager. +This file `pnpm-lock.yaml` for `pnpm` and `packages-lock.json` for `npm` will be +symlinked to the `var/instance/assets/` directory. + +Hints +===== + +`uv` + +The development with `uv` is a little bit different than with `pipenv`. If there +is a `uv.lock` file packages have to be updated manually or by removing the +`uv.lock` file. The absence of the `uv.lock` file triggeres a new dependency +resolving call which takes into account of new released packages. It would also +be possible to use the `uv sync --upgrade` feature of `uv` but this installs +also beta versions of packages which is not recommended. It may sound strange to +remove the `uv.lock` file, but `uv` is that fast that deleting the `.venv` +directory and the `uv.lock` file is the easiest and fastest and safest way to +upgrade the packages. + +UV uses a local `.venv` directory. This will be created automatically, if not +existing in the same directory as the `pyproject.toml` file. + +UV uses a `pyproject.toml` file. + +`rpc-server` + +To use `pnpm` with the `rpc-server` it is necessary to add +`WEBPACKEXT_NPM_PKG_CLS = "pynpm:PNPMPackage"` to the `invenio.cfg` file. + Containerized 'Production' environment -------------------------------------- diff --git a/invenio_cli/commands/assets.py b/invenio_cli/commands/assets.py index d07ac1c..c83718a 100644 --- a/invenio_cli/commands/assets.py +++ b/invenio_cli/commands/assets.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2026 California Institute of Technology. # SPDX-FileCopyrightText: 2020 CERN. +# SPDX-FileCopyrightText: 2025 Graz University of Technology. # SPDX-License-Identifier: MIT """Invenio module to ease the creation and management of applications.""" diff --git a/invenio_cli/commands/containers.py b/invenio_cli/commands/containers.py index a66fd3b..d454afc 100644 --- a/invenio_cli/commands/containers.py +++ b/invenio_cli/commands/containers.py @@ -137,7 +137,7 @@ def _setup(self, project_shortname="/opt/var/instance/"): ), ] - if rdm_version()[0] >= 10: + if rdm_version(self.cli_config)[0] >= 10: steps.extend( [ FunctionStep( @@ -159,11 +159,11 @@ def _setup(self, project_shortname="/opt/var/instance/"): ] ) - if rdm_version()[0] >= 11: + if rdm_version(self.cli_config)[0] >= 11: steps.extend(self.rdm_fixtures(project_shortname)) steps.extend(self.translations(project_shortname)) - if rdm_version()[0] >= 12: + if rdm_version(self.cli_config)[0] >= 12: steps.extend(self.declare_queues(project_shortname)) return steps diff --git a/invenio_cli/commands/install.py b/invenio_cli/commands/install.py index 3ce7e68..ac2ec66 100644 --- a/invenio_cli/commands/install.py +++ b/invenio_cli/commands/install.py @@ -5,7 +5,6 @@ """Invenio module to ease the creation and management of applications.""" from ..helpers import filesystem -from ..helpers.process import run_cmd from .local import LocalCommands from .packages import PackagesCommands from .steps import FunctionStep @@ -33,17 +32,16 @@ def install_py_dependencies(self, pre, dev=False): def update_instance_path(self): """Update path to instance in config.""" - result = run_cmd( - self.cli_config.python_package_manager.run_command( - "invenio", - "shell", - # make sure the shell does not append cursor if editing_mode is set to `vi` in config - "--TerminalInteractiveShell.editing_mode=''", - "--no-term-title", - "-c", - "\"print(app.instance_path, end='')\"", - ) + op = self.cli_config.python_package_manager.invenio_command( + "invenio", + "shell", + # make sure the shell does not append cursor if editing_mode is set to `vi` in config + "--TerminalInteractiveShell.editing_mode=''", + "--no-term-title", + "-c", + "print(app.instance_path, end='')", ) + result = op(capture=True) if result.status_code == 0: self.cli_config.update_instance_path(result.output.strip()) result.output = "Instance path updated successfully." diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index b45691e..9f0e757 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2026 California Institute of Technology. # SPDX-FileCopyrightText: 2020 CERN. -# SPDX-FileCopyrightText: 2022 Graz University of Technology. +# SPDX-FileCopyrightText: 2022-2026 Graz University of Technology. # SPDX-FileCopyrightText: 2026 Northwestern University. # SPDX-License-Identifier: MIT @@ -17,7 +17,9 @@ import click from ..helpers import env, filesystem -from ..helpers.process import ProcessResponse, run_interactive +from ..helpers.package_managers import LocalOp +from ..helpers.process import ProcessResponse +from ..helpers.rpc import RPCOp from ..helpers.versions import rdm_version from .commands import Commands @@ -106,22 +108,21 @@ def update_statics_and_assets(self, force, debug=False, log_file=None): # Commands py_pkg_man = self.cli_config.python_package_manager js_pkg_man = self.cli_config.javascript_package_manager - ops = [py_pkg_man.run_command("invenio", "collect", "--verbose")] + ops = [py_pkg_man.invenio_command("invenio", "collect", "--verbose")] if force: - ops.append(py_pkg_man.run_command("invenio", "webpack", "clean", "create")) + ops.append( + py_pkg_man.invenio_command("invenio", "webpack", "clean", "create") + ) # We need to copy the js lock files here, since webpack regenerates # package.json and we want the locked version instead. ops.append(self._copy_js_lock_files) - ops.append(py_pkg_man.run_command("invenio", "webpack", "install")) + ops.append(py_pkg_man.invenio_command("invenio", "webpack", "install")) else: - ops.append(py_pkg_man.run_command("invenio", "webpack", "create")) - # We need to copy the js lock files here, since webpack regenerates - # package.json and we want the locked version instead. + ops.append(py_pkg_man.invenio_command("invenio", "webpack", "create")) ops.append(self._copy_js_lock_files) ops.append(self._statics) - ops.append(py_pkg_man.run_command("invenio", "webpack", "build")) - + ops.append(py_pkg_man.invenio_command("invenio", "webpack", "build")) # Keep the same messages for some of the operations for backward compatibility messages = { "build": "Building assets...", @@ -130,16 +131,15 @@ def update_statics_and_assets(self, force, debug=False, log_file=None): with env(FLASK_DEBUG="1" if debug else "0"): for op in ops: - if callable(op): - response = op() - else: - if op[-1] in messages: - click.secho(messages[op[-1]], fg="green") - response = run_interactive( - op, + if isinstance(op, (LocalOp, RPCOp)): + if op.label in messages: + click.secho(messages[op.label], fg="green") + response = op( env={"PIPENV_VERBOSITY": "-1", **js_pkg_man.env_overrides()}, log_file=log_file, ) + else: + response = op() if response.status_code != 0: break return response @@ -224,7 +224,7 @@ def run_worker( def run_jobs_scheduler(self, celery_log_file=None, celery_log_level="INFO"): """Run Celery beat scheduler for jobs.""" # Jobs scheduler is only available in RDM v13+ - version = rdm_version() + version = rdm_version(self.cli_config) if version is None: click.secho( "RDM version couldn't be determined. Not running jobs scheduler.", diff --git a/invenio_cli/commands/services.py b/invenio_cli/commands/services.py index 182bf31..7387401 100644 --- a/invenio_cli/commands/services.py +++ b/invenio_cli/commands/services.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2020-2024 CERN. # SPDX-FileCopyrightText: 2021 Esteban J. G. Gabancho. -# SPDX-FileCopyrightText: 2024 Graz University of Technology. +# SPDX-FileCopyrightText: 2024-2025 Graz University of Technology. # SPDX-License-Identifier: MIT """Invenio module to ease the creation and management of applications.""" @@ -38,7 +38,12 @@ def ensure_containers_running(self): cmd_env = {"INSTANCE_PATH": str(instance_path)} # Set environment variable for the instance path, it might be needed by docker services with env(**cmd_env): - self.docker_helper.start_containers() + response = self.docker_helper.start_containers() + if response.status_code != 0: + return ProcessResponse( + error="Failed to start containers (see docker compose output above).", + status_code=1, + ) services = ["redis", self.cli_config.get_db_type(), "search"] for service in services: @@ -84,7 +89,7 @@ def _cleanup(self): pkg_man = self.cli_config.python_package_manager steps = [ CommandStep( - cmd=pkg_man.run_command( + cmd=pkg_man.invenio_command( "invenio", "shell", "--no-term-title", @@ -96,13 +101,13 @@ def _cleanup(self): skippable=True, ), CommandStep( - cmd=pkg_man.run_command("invenio", "db", "destroy", "--yes-i-know"), + cmd=pkg_man.invenio_command("invenio", "db", "destroy", "--yes-i-know"), env={"PIPENV_VERBOSITY": "-1"}, message="Destroying database...", skippable=True, ), CommandStep( - cmd=pkg_man.run_command( + cmd=pkg_man.invenio_command( "invenio", "index", "destroy", @@ -114,7 +119,9 @@ def _cleanup(self): skippable=True, ), CommandStep( - cmd=pkg_man.run_command("invenio", "index", "queue", "init", "purge"), + cmd=pkg_man.invenio_command( + "invenio", "index", "queue", "init", "purge" + ), env={"PIPENV_VERBOSITY": "-1"}, message="Purging queues...", skippable=True, @@ -132,7 +139,7 @@ def _default_location_path(self): """Build default location path based on file storage selection.""" file_storage = self.cli_config.get_file_storage() if file_storage == "local": - return "{}/data".format(self.cli_config.get_instance_path()) + return "{}/data".format(self.cli_config.get_data_path()) return "{}://default".format(self.cli_config.get_file_storage().lower()) def _setup(self, demo_data=False): @@ -145,12 +152,12 @@ def _setup(self, demo_data=False): message="Checking services are not setup...", ), CommandStep( - cmd=pkg_man.run_command("invenio", "db", "init", "create"), + cmd=pkg_man.invenio_command("invenio", "db", "init", "create"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating database...", ), CommandStep( - cmd=pkg_man.run_command( + cmd=pkg_man.invenio_command( "invenio", "files", "location", @@ -163,12 +170,12 @@ def _setup(self, demo_data=False): message="Creating files location...", ), CommandStep( - cmd=pkg_man.run_command("invenio", "roles", "create", "admin"), + cmd=pkg_man.invenio_command("invenio", "roles", "create", "admin"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating admin role...", ), CommandStep( - cmd=pkg_man.run_command( + cmd=pkg_man.invenio_command( "invenio", "access", "allow", @@ -180,19 +187,19 @@ def _setup(self, demo_data=False): message="Allowing superuser access to admin role...", ), CommandStep( - cmd=pkg_man.run_command("invenio", "index", "init"), + cmd=pkg_man.invenio_command("invenio", "index", "init"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating indices...", ), ] - rdm_version_value = rdm_version() + rdm_version_value = rdm_version(self.cli_config) if rdm_version_value: if rdm_version_value[0] >= 10: steps.extend( [ CommandStep( - cmd=pkg_man.run_command( + cmd=pkg_man.invenio_command( "invenio", "rdm-records", "custom-fields", @@ -202,7 +209,7 @@ def _setup(self, demo_data=False): message="Creating custom fields for records...", ), CommandStep( - cmd=pkg_man.run_command( + cmd=pkg_man.invenio_command( "invenio", "communities", "custom-fields", @@ -232,9 +239,10 @@ def _setup(self, demo_data=False): ) if ils_version(): - cmd = pkg_man.run_command("invenio", "setup", "--verbose") + args = ["invenio", "setup", "--verbose"] if not demo_data: - cmd.append("--skip-demo-data") + args.append("--skip-demo-data") + cmd = pkg_man.invenio_command(*args) steps.extend( [ CommandStep( @@ -260,7 +268,7 @@ def demo(self): pkg_man = self.cli_config.python_package_manager steps = [ CommandStep( - cmd=pkg_man.run_command("invenio", "rdm-records", "demo"), + cmd=pkg_man.invenio_command("invenio", "rdm-records", "demo"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating demo records...", ) @@ -271,14 +279,14 @@ def demo(self): def declare_queues(self): """Steps to declare the MQ queues required for statistics, etc.""" pkg_man = self.cli_config.python_package_manager - command = pkg_man.run_command("invenio", "queues", "declare") + command = pkg_man.invenio_command("invenio", "queues", "declare") steps = [CommandStep(cmd=command, message="Declaring queues...")] return steps def fixtures(self): """Steps to set up the required fixtures for the instance.""" pkg_man = self.cli_config.python_package_manager - command = pkg_man.run_command("invenio", "rdm-records", "fixtures") + command = pkg_man.invenio_command("invenio", "rdm-records", "fixtures") steps = [ CommandStep( cmd=command, @@ -292,7 +300,7 @@ def fixtures(self): def rdm_fixtures(self): """Steps to set up the rdm fixtures for the instance.""" pkg_man = self.cli_config.python_package_manager - command = pkg_man.run_command("invenio", "rdm", "fixtures") + command = pkg_man.invenio_command("invenio", "rdm", "fixtures") steps = [ CommandStep( cmd=command, diff --git a/invenio_cli/commands/steps.py b/invenio_cli/commands/steps.py index 3eedfd6..2fa72d0 100644 --- a/invenio_cli/commands/steps.py +++ b/invenio_cli/commands/steps.py @@ -57,5 +57,11 @@ def __init__(self, cmd, env=None, log_file=None, **kwargs): self.log_file = log_file def execute(self): - """Execute the function with the given arguments.""" + """Execute the command, an op (RPCOp/LocalOp) or a plain argv list.""" + if callable(self.cmd): + response = self.cmd(env=self.env, log_file=self.log_file) + if response.status_code > 0 and self.skippable: + response.warning = True + response.status_code = 0 + return response return run_interactive(self.cmd, self.env, self.skippable, self.log_file) diff --git a/invenio_cli/helpers/cli_config.py b/invenio_cli/helpers/cli_config.py index 3592cbe..9f20465 100644 --- a/invenio_cli/helpers/cli_config.py +++ b/invenio_cli/helpers/cli_config.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2019-2024 CERN. # SPDX-FileCopyrightText: 2019-2020 Northwestern University. # SPDX-FileCopyrightText: 2021 Esteban J. G. Gabancho. -# SPDX-FileCopyrightText: 2024 Graz University of Technology. +# SPDX-FileCopyrightText: 2024-2026 Graz University of Technology. # SPDX-License-Identifier: MIT """Invenio-cli configuration file.""" @@ -70,15 +70,23 @@ def __init__(self, project_dir="./"): def python_package_manager(self) -> PythonPackageManager: """Get python packages manager.""" manager_name = self.config[CLIConfig.CLI_SECTION].get("python_package_manager") + use_rpc = self.config[CLIConfig.CLI_SECTION].getboolean( + "use_rpc", fallback=False + ) + # passed as a callable: the socket lives in the instance directory, + # which install only discovers partway through, and RPC should kick + # in from that moment on + get_socket_path = self.get_rpc_socket_path if use_rpc else None + if manager_name == Pipenv.name: - return Pipenv() + return Pipenv(get_rpc_socket_path=get_socket_path) elif manager_name == UV.name: - return UV() + return UV(get_rpc_socket_path=get_socket_path) if (self.project_path / "Pipfile").is_file(): - return Pipenv() + return Pipenv(get_rpc_socket_path=get_socket_path) elif (self.project_path / "pyproject.toml").is_file(): - return UV() + return UV(get_rpc_socket_path=get_socket_path) else: raise RuntimeError( "Could not determine the Python package manager, please configure it." @@ -101,6 +109,14 @@ def get_project_dir(self): """Returns path to project directory.""" return self.config_path.parent.resolve() + def get_data_path(self): + """Return path to data.""" + path = self.private_config[CLIConfig.CLI_SECTION].get("data_path") + if path: + return Path(path) + else: + return self.get_instance_path() + def get_instance_path(self, throw=True): """Returns path to application instance directory. @@ -146,6 +162,12 @@ def get_project_shortname(self): """Returns the project's shortname.""" return self.config[CLIConfig.COOKIECUTTER_SECTION]["project_shortname"] + def get_rpc_socket_path(self): + """Returns the RPC socket path (None until the instance path is set).""" + instance_path = self.get_instance_path(throw=False) + if instance_path: + return instance_path / "rpc.sock" + def get_search_port(self): """Returns the search port.""" return self.private_config[CLIConfig.CLI_SECTION].get("search_port", "9200") @@ -165,6 +187,10 @@ def get_web_host(self): """Returns web host.""" return self.private_config[CLIConfig.CLI_SECTION].get("web_host", "127.0.0.1") + def get_app_rdm_version(self): + """Returns app rdm version.""" + return self.private_config[CLIConfig.CLI_SECTION].get("app_rdm", None) + def get_db_type(self): """Returns the database type (mysql, postgresql).""" return self.config[CLIConfig.COOKIECUTTER_SECTION]["database"] diff --git a/invenio_cli/helpers/docker_helper.py b/invenio_cli/helpers/docker_helper.py index 95343f2..5d67837 100644 --- a/invenio_cli/helpers/docker_helper.py +++ b/invenio_cli/helpers/docker_helper.py @@ -90,7 +90,8 @@ def start_containers(self, app_only=False): if app_only: command.extend(["web-ui", "web-api"]) - return run_cmd(command) + # interactive so image pulls and container creation stream live + return run_interactive(command) def stop_containers(self): """Stop currently running containers.""" diff --git a/invenio_cli/helpers/package_managers.py b/invenio_cli/helpers/package_managers.py index 6e69a69..76877ff 100644 --- a/invenio_cli/helpers/package_managers.py +++ b/invenio_cli/helpers/package_managers.py @@ -1,4 +1,5 @@ # SPDX-FileCopyrightText: 2025 TU Wien. +# SPDX-FileCopyrightText: 2025-2026 Graz University of Technology. # SPDX-License-Identifier: MIT """Wrappers around various package managers to be used under the hood.""" @@ -13,6 +14,30 @@ from pynpm import NPMPackage, PNPMPackage from ..helpers.process import ProcessResponse +from .process import run_cmd, run_interactive +from .rpc import RPCClient, RPCOp + + +class LocalOp: + """Executable invenio command op that runs as a subprocess. + + The RPC-less counterpart of ``RPCOp``, with the same call shape. + """ + + def __init__(self, argv): + """Construct.""" + self.argv = list(argv) + + @property + def label(self): + """Subcommand name, used for progress messages.""" + return self.argv[-1] + + def __call__(self, env=None, log_file=None, capture=False): + """Run the command and return a ProcessResponse.""" + if capture: + return run_cmd(self.argv) + return run_interactive(self.argv, env=env, log_file=log_file) class PythonPackageManager(ABC): @@ -21,6 +46,40 @@ class PythonPackageManager(ABC): name: str = None lock_file_name: str = None + def __init__(self, get_rpc_socket_path=None): + """Construct. + + ``get_rpc_socket_path`` is a zero-argument callable returning the + RPC socket path, or None while it is unknown. A callable because + the socket lives in the instance directory, which is only + discovered partway through the first install. + """ + self._get_rpc_socket_path = get_rpc_socket_path + self.rpc = None + + def invenio_command(self, *command): + """Build an executable op for the given invenio CLI command. + + Returns an ``RPCOp`` when the RPC server is enabled and its socket + path is known, otherwise a ``LocalOp`` running the command as a + subprocess, both with the same call shape. Only use this for + short-lived, non-interactive ``invenio`` commands; anything else + goes through ``run_command``. + """ + if self.rpc is None and self._get_rpc_socket_path: + socket_path = self._get_rpc_socket_path() + if socket_path: + self.rpc = RPCClient( + socket_path, + self.run_command( + "invenio", "rpc-server", "start", "--socket", str(socket_path) + ), + ) + if self.rpc: + # drop the leading "invenio"; the server routes the argv itself + return RPCOp(self.rpc, command[1:]) + return LocalOp(self.run_command(*command)) + def run_command(self, *command: str) -> List[str]: """Generate command to run the given command in the managed environment.""" raise NotImplementedError() diff --git a/invenio_cli/helpers/rpc.py b/invenio_cli/helpers/rpc.py new file mode 100644 index 0000000..5762c70 --- /dev/null +++ b/invenio_cli/helpers/rpc.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: 2025 Graz University of Technology. +# SPDX-FileCopyrightText: 2026 CERN. +# SPDX-License-Identifier: MIT + +"""Client for the invenio RPC server. + +The server (``invenio rpc-server start``) listens on a Unix domain socket +and runs invenio CLI commands in a long-lived process, so each command +skips the Python startup and app creation cost. The protocol is one JSON +line per request (``{"argv": [...]}``) answered by one JSON line. By +default the client passes its stdout/stderr file descriptors with the +request (SCM_RIGHTS), the server streams the command output straight to +them, and the response carries only the exit code; without descriptors +the response carries the captured output instead. +""" + +import array +import atexit +import json +import os +import socket +import sys +import time +from subprocess import Popen, TimeoutExpired + +from .process import ProcessResponse + +CONNECT_TIMEOUT = 5 +STARTUP_TIMEOUT = 120 # starting the server includes a full app creation + + +class RPCClient: + """Talks to the invenio RPC server over its Unix domain socket. + + If no server is listening, one is spawned on first use and terminated + again when invenio-cli exits; a server the user started themselves is + used but never owned. + """ + + def __init__(self, socket_path, start_command): + """Construct.""" + self.socket_path = str(socket_path) + self.start_command = start_command + self.server = None + + def request(self, payload, fds=None): + """Send one JSON line (and fds) and read back one JSON-line response.""" + line = json.dumps(payload).encode("utf-8") + b"\n" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.settimeout(CONNECT_TIMEOUT) + s.connect(self.socket_path) + if fds: + s.sendmsg( + [line], + [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))], + ) + else: + s.sendall(line) + # commands may run for a long time (e.g. webpack builds) + s.settimeout(None) + with s.makefile("rb") as f: + return json.loads(f.readline()) + + def listening(self): + """Return whether something accepts connections on the socket. + + A connect-only probe does not need the server to respond, so it + also recognizes a server that is busy running a long command. + """ + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.settimeout(CONNECT_TIMEOUT) + s.connect(self.socket_path) + return True + except OSError: + return False + + def ensure_running(self, env=None): + """Spawn a server and wait until its socket accepts connections. + + Returns an error message, or None on success. The server binds the + socket only after its app is fully created, so accepting + connections means ready. ``env`` is added to the spawned server's + environment, so it lands in the app config the server builds at + startup (e.g. ``INVENIO_*`` variables). + """ + self.server = Popen(self.start_command, env={**os.environ, **(env or {})}) + atexit.register(self.shutdown) + deadline = time.monotonic() + STARTUP_TIMEOUT + while time.monotonic() < deadline: + if self.listening(): + return None + if self.server.poll() is not None: + # our spawn may have lost a race against another invenio-cli + # whose server now owns the socket — that is still a success + if self.listening(): + return None + return ( + "RPC server exited with code " + f"{self.server.returncode} during startup." + ) + time.sleep(0.2) + + self.shutdown() + return f"RPC server did not start within {STARTUP_TIMEOUT} seconds." + + def call(self, argv, env=None, log_file=None, capture=False): + """Run an invenio CLI command on the server. + + The request is sent directly: a busy single-threaded server queues + the connection until it is free, and only a refused or missing + socket makes the client spawn a server (with ``env`` applied to it) + and retry once. By default the command's output streams live to + this process' stdout/stderr (or into ``log_file``) via the passed + descriptors; with ``capture`` it is returned on the response. + """ + try: + try: + return self.request_command(argv, log_file, capture) + except (FileNotFoundError, ConnectionRefusedError): + # nothing is listening on the socket + error = self.ensure_running(env=env) + if error: + return ProcessResponse(error=error, status_code=1) + return self.request_command(argv, log_file, capture) + except (OSError, ValueError) as e: + return ProcessResponse(error=f"RPC request failed: {e}", status_code=1) + + def request_command(self, argv, log_file=None, capture=False): + """Send one command request to a listening server.""" + payload = {"argv": list(argv)} + if capture: + response = self.request(payload) + return ProcessResponse( + output=response["stdout"], + error=response["stderr"], + status_code=response["exit_code"], + ) + + # flush so our own pending output stays ordered with the + # command output the server writes to the same descriptors + sys.stdout.flush() + sys.stderr.flush() + if log_file: + with open(log_file, "ab") as f: + response = self.request(payload, fds=[f.fileno(), f.fileno()]) + else: + response = self.request( + payload, fds=[sys.stdout.fileno(), sys.stderr.fileno()] + ) + return ProcessResponse( + error=response.get("stderr"), + status_code=response["exit_code"], + ) + + def shutdown(self): + """Terminate the server if this process spawned it.""" + if self.server is None or self.server.poll() is not None: + return + self.server.terminate() + try: + self.server.wait(timeout=10) + except TimeoutExpired: + self.server.kill() + + +class RPCOp: + """Executable invenio command op that runs on the RPC server. + + Has the same call shape as ``LocalOp`` so callers never branch on + which one they got. + """ + + def __init__(self, client, argv): + """Construct.""" + self.client = client + self.argv = list(argv) + + @property + def label(self): + """Subcommand name, used for progress messages.""" + return self.argv[-1] + + def __call__(self, env=None, log_file=None, capture=False): + """Execute on the RPC server. + + ``env`` reaches the server's environment (and thus its app config) + only when this call is the one that spawns it; a server that is + already running keeps the environment it was started with. + """ + return self.client.call(self.argv, env=env, log_file=log_file, capture=capture) diff --git a/invenio_cli/helpers/versions.py b/invenio_cli/helpers/versions.py index ab17f93..fb99bbc 100644 --- a/invenio_cli/helpers/versions.py +++ b/invenio_cli/helpers/versions.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2022-2025 CERN. # SPDX-FileCopyrightText: 2025 TU Wien. +# SPDX-FileCopyrightText: 2025 Graz University of Technology. # SPDX-License-Identifier: MIT """Invenio CLI dependencies helper.""" @@ -53,9 +54,17 @@ def _from_pyproject_toml(dep_name): return _parse_version(v.version) -def rdm_version(): - """Return the latest RDM version.""" - if os.path.isfile("./Pipfile"): +def rdm_version(cli_config=None): + """Return the latest RDM version. + + A version pinned via the ``app_rdm`` option takes precedence, but + ``cli_config`` is optional: callers without a project context (e.g. + ``check-requirements``) fall back to the dependency files. + """ + if cli_config and (app_rdm_version := cli_config.get_app_rdm_version()): + return [int(v) for v in app_rdm_version.split(".")] + + elif os.path.isfile("./Pipfile"): return _from_pipfile("invenio-app-rdm") elif os.path.isfile("./pyproject.toml"): diff --git a/tests/test_rpc.py b/tests/test_rpc.py new file mode 100644 index 0000000..7971be8 --- /dev/null +++ b/tests/test_rpc.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: 2026 CERN. +# SPDX-License-Identifier: MIT + +"""RPC client tests.""" + +import array +import json +import os +import socket +import socketserver +import sys +import tempfile +import threading +import time +from pathlib import Path + +import pytest + +from invenio_cli.commands.steps import CommandStep +from invenio_cli.helpers.package_managers import UV, LocalOp +from invenio_cli.helpers.rpc import RPCClient, RPCOp + + +def _recv_request(sock): + """Read one JSON line plus any file descriptors sent with it.""" + buf = b"" + fds = [] + fd_size = array.array("i").itemsize + while b"\n" not in buf: + data, ancdata, _, _ = sock.recvmsg(4096, socket.CMSG_SPACE(2 * fd_size)) + if not data: + break + for level, ctype, cdata in ancdata: + if level == socket.SOL_SOCKET and ctype == socket.SCM_RIGHTS: + received = array.array("i") + received.frombytes(cdata[: len(cdata) - (len(cdata) % fd_size)]) + fds.extend(received) + buf += data + return buf, fds + + +class FakeInvenioHandler(socketserver.BaseRequestHandler): + """Respond to the RPC protocol like the invenio RPC server.""" + + def handle(self): + """Answer one JSON-line request with one JSON-line response.""" + line, fds = _recv_request(self.request) + if not line: + return # connect-only liveness probe + request = json.loads(line) + if request.get("argv") == ["slow"]: + time.sleep(0.3) # simulate a long-running command + if fds: + os.write(fds[0], b"streamed out\n") + os.write(fds[1], b"streamed err\n") + for fd in fds: + os.close(fd) + response = {"exit_code": 7} + else: + response = { + "exit_code": 7, + "stdout": " ".join(request["argv"]), + "stderr": "warning\n", + } + self.request.sendall(json.dumps(response).encode("utf-8") + b"\n") + + +@pytest.fixture() +def short_tmp_path(): + """A directory short enough for AF_UNIX paths (~104 bytes on macOS).""" + with tempfile.TemporaryDirectory(prefix="rpc", dir="/tmp") as tmp: + yield Path(tmp) + + +@pytest.fixture() +def rpc_socket(short_tmp_path): + """A running protocol server on a temporary socket.""" + socket_path = short_tmp_path / "rpc.sock" + server = socketserver.UnixStreamServer(str(socket_path), FakeInvenioHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield socket_path + server.shutdown() + server.server_close() + + +def test_call_forwards_output_to_our_descriptors(rpc_socket, capfd): + """By default the command output streams to our stdout/stderr.""" + client = RPCClient(rpc_socket, start_command=None) + response = client.call(["webpack", "build"]) + assert response.status_code == 7 + captured = capfd.readouterr() + assert captured.out == "streamed out\n" + assert captured.err == "streamed err\n" + + +def test_call_forwards_output_to_a_log_file(rpc_socket, short_tmp_path): + """With a log file, the command output lands there instead.""" + log_file = short_tmp_path / "build.log" + client = RPCClient(rpc_socket, start_command=None) + response = client.call(["webpack", "build"], log_file=str(log_file)) + assert response.status_code == 7 + assert log_file.read_text() == "streamed out\nstreamed err\n" + + +def test_call_captures_output_on_request(rpc_socket): + """With capture, output comes back on the ProcessResponse.""" + client = RPCClient(rpc_socket, start_command=None) + response = client.call(["webpack", "build"], capture=True) + assert response.output == "webpack build" + assert response.error == "warning\n" + assert response.status_code == 7 + + +# A stand-in for "invenio rpc-server start": serves the captured-output +# protocol on the socket given as first argument until it is terminated. +# Echoes RPC_TEST_VAR so tests can check the environment it was spawned with. +SERVER_SCRIPT = """ +import json, os, socketserver, sys + +class Handler(socketserver.StreamRequestHandler): + def handle(self): + line = self.rfile.readline() + if not line: + return # connect-only liveness probe + response = { + "exit_code": 0, + "stdout": os.environ.get("RPC_TEST_VAR", "ok"), + "stderr": "", + } + self.wfile.write(json.dumps(response).encode("utf-8") + b"\\n") + +with socketserver.UnixStreamServer(sys.argv[1], Handler) as server: + server.serve_forever() +""" + + +def test_call_spawns_and_terminates_a_server(short_tmp_path): + """Without a listening server, one is spawned, used and shut down.""" + socket_path = short_tmp_path / "rpc.sock" + client = RPCClient( + socket_path, [sys.executable, "-c", SERVER_SCRIPT, str(socket_path)] + ) + response = client.call(["collect"], capture=True) + assert response.status_code == 0 + assert response.output == "ok" + client.shutdown() + assert client.server.poll() is not None + + +def test_call_passes_env_to_the_spawned_server(short_tmp_path): + """env reaches the server's environment when this call spawns it.""" + socket_path = short_tmp_path / "rpc.sock" + client = RPCClient( + socket_path, [sys.executable, "-c", SERVER_SCRIPT, str(socket_path)] + ) + response = client.call(["collect"], env={"RPC_TEST_VAR": "hello"}, capture=True) + assert response.output == "hello" + client.shutdown() + + +def test_call_reports_a_server_that_dies_during_startup(short_tmp_path): + """A start command that exits early fails the call instead of hanging.""" + client = RPCClient( + short_tmp_path / "rpc.sock", [sys.executable, "-c", "raise SystemExit(3)"] + ) + response = client.call(["collect"]) + assert response.status_code == 1 + assert "exited with code 3" in response.error + + +def test_spawn_race_loser_uses_the_listening_server(rpc_socket): + """A dead spawn is fine when another server owns the socket.""" + client = RPCClient(rpc_socket, ["sh", "-c", "exit 9"]) + assert client.ensure_running() is None + + +def test_concurrent_calls_queue_behind_a_busy_server(rpc_socket): + """Calls connect directly and wait their turn instead of respawning.""" + client = RPCClient(rpc_socket, start_command=None) + results = [] + + def run(): + """Issue one slow captured call.""" + results.append(client.call(["slow"], capture=True).status_code) + + threads = [threading.Thread(target=run) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert results == [7, 7] + + +def test_invenio_command_builds_an_rpc_op(short_tmp_path): + """With a socket path, invenio_command defers to the RPC server.""" + manager = UV(get_rpc_socket_path=lambda: short_tmp_path / "rpc.sock") + op = manager.invenio_command("invenio", "webpack", "build") + assert isinstance(op, RPCOp) + assert op.argv == ["webpack", "build"] + assert op.label == "build" + + +def test_invenio_command_picks_up_a_late_socket_path(short_tmp_path): + """RPC kicks in once install discovers the instance path.""" + socket_path = None + manager = UV(get_rpc_socket_path=lambda: socket_path) + assert isinstance(manager.invenio_command("invenio", "collect"), LocalOp) + socket_path = short_tmp_path / "rpc.sock" + assert isinstance(manager.invenio_command("invenio", "collect"), RPCOp) + + +def test_invenio_command_without_rpc_builds_a_local_op(): + """Without a socket path, invenio_command wraps the plain command.""" + op = UV().invenio_command("invenio", "collect") + assert isinstance(op, LocalOp) + assert op.argv == ["uv", "run", "--no-sync", "invenio", "collect"] + assert op.label == "collect" + + +def test_local_op_runs_a_subprocess(): + """LocalOp runs the argv and reports the real exit code.""" + assert LocalOp(["sh", "-c", "exit 3"])().status_code == 3 + assert LocalOp(["sh", "-c", "echo hi"])(capture=True).output == "hi\n" + + +def test_command_step_executes_an_op(rpc_socket, capfd): + """CommandStep runs an op, with output forwarded and the exit code kept.""" + client = RPCClient(rpc_socket, start_command=None) + step = CommandStep(cmd=RPCOp(client, ["db", "init", "create"])) + response = step.execute() + assert response.status_code == 7 + assert capfd.readouterr().out == "streamed out\n" + + +def test_command_step_skippable_turns_op_failure_into_warning(rpc_socket, capfd): + """A skippable step reports a failed RPC command as a warning.""" + client = RPCClient(rpc_socket, start_command=None) + step = CommandStep(cmd=RPCOp(client, ["db", "destroy"]), skippable=True) + response = step.execute() + assert response.status_code == 0 + assert response.warning is True diff --git a/tests/test_versions.py b/tests/test_versions.py new file mode 100644 index 0000000..197f0d8 --- /dev/null +++ b/tests/test_versions.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2026 CERN. +# SPDX-License-Identifier: MIT + +"""Version helper tests.""" + +from invenio_cli.helpers.cli_config import CLIConfig +from invenio_cli.helpers.versions import rdm_version + + +def test_rdm_version_without_cli_config_reads_dependency_files(tmp_path, monkeypatch): + """Callers without a project context (check-requirements) stay valid.""" + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "site"\ndependencies = ["invenio-app-rdm~=13.0.0"]\n' + ) + monkeypatch.chdir(tmp_path) + assert rdm_version() == [13, 0, 0] + + +def test_rdm_version_prefers_the_pinned_version(tmp_path, monkeypatch): + """An app_rdm pin in the private config wins over dependency files.""" + (tmp_path / ".invenio").write_text("[cli]\n") + (tmp_path / ".invenio.private").write_text("[cli]\napp_rdm = 14.1.0\n") + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "site"\ndependencies = ["invenio-app-rdm~=13.0.0"]\n' + ) + monkeypatch.chdir(tmp_path) + assert rdm_version(CLIConfig(tmp_path)) == [14, 1, 0]