diff --git a/CONSTRUCT.md b/CONSTRUCT.md index 17a71302f..ace5183bf 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -235,6 +235,7 @@ The type of the installer being created. Possible values are: - `sh`: shell-based installer for Linux or macOS - `pkg`: macOS GUI installer built with Apple's `pkgbuild` - `exe`: Windows GUI installer built with NSIS +- `docker`: generates a Dockerfile that replicates the installation environment (Does not build image. See `docker_build` option below.) The default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well @@ -679,6 +680,29 @@ freeze_base: message: "This base environment is frozen and cannot be modified." ``` +### `docker_base_image` + +Base image to use for docker builds when `installer_type` includes `docker` or `docker_build` is True. +Should be a specific image reference. For reproducibility, please specify a SHA256 digest. +For example: `debian:13.4-slim@sha256:abc123...`. If the digest is not provided, a warning will be shown. + +### `docker_tag` + +Tag to use for the docker image. +If not provided, it will default to `:`. + +### `docker_labels` + +Additional labels to add to the built docker image. +The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` are +set automatically from `name` and `version`. + +### `docker_build` + +If `True`, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball. +``--.tar`` will be created in the output docker directory. +Requires `docker_base_image` to be specified. + ## Available selectors - `aarch64` diff --git a/constructor/_schema.py b/constructor/_schema.py index 87945ad03..bde37d452 100644 --- a/constructor/_schema.py +++ b/constructor/_schema.py @@ -42,6 +42,7 @@ class InstallerTypes(StrEnum): EXE = "exe" PKG = "pkg" SH = "sh" + DOCKER = "docker" class PkgDomains(StrEnum): @@ -403,6 +404,7 @@ class ConstructorConfiguration(BaseModel): - `sh`: shell-based installer for Linux or macOS - `pkg`: macOS GUI installer built with Apple's `pkgbuild` - `exe`: Windows GUI installer built with NSIS + - `docker`: generates a Dockerfile that replicates the installation environment (Does not build image. See `docker_build` option below.) The default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well @@ -853,6 +855,29 @@ class ConstructorConfiguration(BaseModel): message: "This base environment is frozen and cannot be modified." ``` """ + docker_base_image: Annotated[str, Field(min_length=1)] | None = None + """ + Base image to use for docker builds when `installer_type` includes `docker` or `docker_build` is True. + Should be a specific image reference. For reproducibility, please specify a SHA256 digest. + For example: `debian:13.4-slim@sha256:abc123...`. If the digest is not provided, a warning will be shown. + """ + docker_tag: NonEmptyStr | None = None + """ + Tag to use for the docker image. + If not provided, it will default to `:`. + """ + docker_labels: dict[NonEmptyStr, NonEmptyStr] = {} + """ + Additional labels to add to the built docker image. + The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` are + set automatically from `name` and `version`. + """ + docker_build: bool = False + """ + If `True`, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball. + ``--.tar`` will be created in the output docker directory. + Requires `docker_base_image` to be specified. + """ def fix_descriptions(obj): diff --git a/constructor/data/construct.schema.json b/constructor/data/construct.schema.json index f0178d738..5f6a6c2c1 100644 --- a/constructor/data/construct.schema.json +++ b/constructor/data/construct.schema.json @@ -245,7 +245,8 @@ "all", "exe", "pkg", - "sh" + "sh", + "docker" ], "title": "InstallerTypes", "type": "string" @@ -638,6 +639,53 @@ "description": "Set default installation prefix for domain users. If not provided, the installation prefix for domain users will be `%LOCALAPPDATA%\\`. By default, it is different from the `default_prefix` value to avoid installing the distribution into the roaming profile. Environment variables will be expanded at install time. Windows only.", "title": "Default Prefix Domain User" }, + "docker_base_image": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Base image to use for docker builds when `installer_type` includes `docker` or `docker_build` is True. Should be a specific image reference. For reproducibility, please specify a SHA256 digest. For example: `debian:13.4-slim@sha256:abc123...`. If the digest is not provided, a warning will be shown.", + "title": "Docker Base Image" + }, + "docker_build": { + "default": false, + "description": "If `True`, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball. ``--.tar`` will be created in the output docker directory. Requires `docker_base_image` to be specified.", + "title": "Docker Build", + "type": "boolean" + }, + "docker_labels": { + "additionalProperties": { + "minLength": 1, + "type": "string" + }, + "default": {}, + "description": "Additional labels to add to the built docker image. The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` are set automatically from `name` and `version`.", + "propertyNames": { + "minLength": 1 + }, + "title": "Docker Labels", + "type": "object" + }, + "docker_tag": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Tag to use for the docker image. If not provided, it will default to `:`.", + "title": "Docker Tag" + }, "environment": { "anyOf": [ { @@ -864,7 +912,7 @@ } ], "default": null, - "description": "The type of the installer being created. Possible values are:\n- `sh`: shell-based installer for Linux or macOS\n- `pkg`: macOS GUI installer built with Apple's `pkgbuild`\n- `exe`: Windows GUI installer built with NSIS\nThe default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well as `sh` on Linux and `exe` on Windows.", + "description": "The type of the installer being created. Possible values are:\n- `sh`: shell-based installer for Linux or macOS\n- `pkg`: macOS GUI installer built with Apple's `pkgbuild`\n- `exe`: Windows GUI installer built with NSIS\n- `docker`: generates a Dockerfile that replicates the installation environment (Does not build image. See `docker_build` option below.)\nThe default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well as `sh` on Linux and `exe` on Windows.", "title": "Installer Type" }, "keep_pkgs": { diff --git a/constructor/docker_build.py b/constructor/docker_build.py new file mode 100644 index 000000000..47349ac78 --- /dev/null +++ b/constructor/docker_build.py @@ -0,0 +1,206 @@ +"""Logic for creating a Dockerfile and/or building portable Docker images from Constructor installers.""" + +import logging +import shutil +import subprocess +import tempfile +from pathlib import Path + +from jinja2 import Template + +from . import __version__ + +logger = logging.getLogger(__name__) + +TEMPLATE_PATH = Path(__file__).parent / "dockerfile_template.tmpl" + +DOCKER_PLATFORM_MAP = { + "linux-64": "linux/amd64", + "linux-aarch64": "linux/arm64", + "linux-armv7l": "linux/arm/v7", + "linux-32": "linux/386", + "linux-ppc64le": "linux/ppc64le", + "linux-s390x": "linux/s390x", + "osx-arm64": "linux/arm64", + "osx-64": "linux/amd64", +} + + +def prepare_docker_context(info: dict, tmp_dir: Path) -> Path: + """Copy the .sh installer into the Docker build directory. + + Parameters + ---------- + info: dict + Constructor installer info dict. Must contain ``_outpath`` and ``_output_dir`` pointing to the built .sh + installer and output directory respectively. + tmp_dir: Path + Path to a temporary directory to stage the Docker build context. The .sh installer will be copied to this directory. + + Returns + ------- + Path + Path to the tmp Docker build directory. + """ + installer_path = Path(info["_outpath"]) + if not installer_path.exists(): + raise RuntimeError(f"Expected .sh installer not found: {installer_path}\n") + + shutil.copy(installer_path, tmp_dir / installer_path.name) + logger.info("Copied installer to tmp directory: %s", tmp_dir / installer_path.name) + + return tmp_dir + + +def generate_dockerfile(info: dict, docker_dir: Path) -> Path: + """ + Render the Dockerfile template and write it to the Docker build directory. + + Parameters + ---------- + info: dict + Constructor installer info dict. + docker_dir: Path + Path to the Docker build directory returned by prepare_docker_context(). + + Returns + ------- + Path + Path to the generated Dockerfile, or None if generation is skipped. + """ + from .conda_interface import MatchSpec + + specs = {MatchSpec(spec).name for spec in info.get("specs", ())} + + docker_template = Template(TEMPLATE_PATH.read_text()) + + docker_base_image = info.get("docker_base_image") + if not docker_base_image: + logger.warning( + "Skipping Dockerfile generation. 'docker_base_image' not specified in construct.yaml." + ) + return None + + if "@" not in docker_base_image: + logger.warning( + "No SHA256 digest specified for docker_base_image. " + "Consider specifying a digest to ensure the integrity of the base image used for the build, e.g.:\n" + " docker_base_image: debian:13.4-slim@sha256:4ffb3a1511099754cddc70eb1b12e50ffdb67619aa0ab6c13fcd800a78ef7c7a\n" + ) + + rendered_dockerfile = docker_template.render( + constructor_version=__version__, + base_image=docker_base_image, + default_prefix=info.get("default_prefix", f"/opt/{info['name'].lower()}"), + installer_filename=Path(info["_outpath"]).name, + clean_cmd="$PREFIX/bin/mamba clean -afy" + if "mamba" in specs + else "$PREFIX/bin/conda clean -afy", + name=info["name"], + version=info["version"], + labels=info.get("docker_labels", {}), + init_cmd="$PREFIX/bin/mamba shell" if "mamba" in specs else "$PREFIX/bin/python -m conda", + register_envs=info.get("register_envs", True), + keep_pkgs=info.get("keep_pkgs", False), + ) + + dockerfile_path = docker_dir / "Dockerfile" + dockerfile_path.write_text(rendered_dockerfile) + logger.info("Dockerfile written to: '%s'", dockerfile_path) + return dockerfile_path + + +def build_image(info: dict, docker_dir: Path) -> None: + """Optionally build the docker image from the generated Dockerfile. + Currently supported on linux and macOS platforms. + + Parameters + ---------- + info: dict + Constructor installer info dict. + docker_dir: Path + Path to the Docker directory containing the Docker outputs. + + """ + if info.get("_platform") not in DOCKER_PLATFORM_MAP: + logger.warning( + f"Building Docker images is not supported on platform '{info['_platform']}'. " + "Skipping Docker build. You can still build the image manually using 'docker buildx' on a supported platform or Docker Desktop. " + "Supported platforms for Docker build are: linux/amd64 and linux/arm64." + ) + return + + if shutil.which("docker") is None: + raise RuntimeError( + "Building a Docker image requires the 'docker' CLI tool to be installed and available in PATH. " + "Install Docker Desktop or Docker Engine to proceed, or " + "use `installer_type: docker` in construct.yaml to " + "generate the Dockerfile without building the image." + ) + + docker_platform = DOCKER_PLATFORM_MAP.get(info["_platform"]) + if docker_platform is None: + raise RuntimeError( + f"Unsupported platform for Docker build: '{info['_platform']}'. " + f"Supported platforms are: {', '.join(DOCKER_PLATFORM_MAP)}." + ) + + tag = info.get("docker_tag", f"{info['name'].lower()}:{info['version'].split('-')[0]}") + docker_dir = Path(info["_output_dir"]) / "docker" + tarball_dest = docker_dir / f"{tag.replace(':', '-')}-{docker_platform.replace('/', '-')}.tar" + + cmd = [ + "docker", + "buildx", + "build", + str(docker_dir), + "--platform", + docker_platform, + "-t", + tag, + "--load", + ] + + logger.info("Building Docker image: '%s'", tag) + subprocess.run(cmd, check=True) + logger.info("Docker image '%s' built successfully.", tag) + + logger.info("Saving Docker image to tarball: '%s'", tarball_dest) + with open(tarball_dest, "wb") as f: + subprocess.run(["docker", "save", tag, "-o", str(tarball_dest)], check=True, stdout=f) + logger.info("Docker image saved to: '%s'", tarball_dest) + + +def create(info: dict, verbose: bool = False) -> None: + """Build a Docker output + + Parameters + ---------- + info: dict + Constructor installer info dict. + verbose: bool, optional + If ``True``, enables verbose logging. + Defaults to ``False``. + + """ + with tempfile.TemporaryDirectory() as temp_dir: + tmp_path = Path(temp_dir) + info["_outpath"] = info["_outpath"].replace(".docker", ".sh") + prepare_docker_context(info, tmp_path) + dockerfile = generate_dockerfile(info, tmp_path) + if dockerfile is None: + logger.warning("Dockerfile generation skipped. Docker image will not be built.") + return + + output_docker_dir = Path(info["_output_dir"]) / "docker" + output_docker_dir.mkdir(parents=True, exist_ok=True) + + shutil.copy(tmp_path / "Dockerfile", output_docker_dir / "Dockerfile") + shutil.copy( + tmp_path / Path(info["_outpath"]).name, output_docker_dir / Path(info["_outpath"]).name + ) + + if info.get("docker_build"): + build_image(info, output_docker_dir) + + logger.info("Docker output complete. Docker directory: '%s'", output_docker_dir) diff --git a/constructor/dockerfile_template.tmpl b/constructor/dockerfile_template.tmpl new file mode 100644 index 000000000..468c11e92 --- /dev/null +++ b/constructor/dockerfile_template.tmpl @@ -0,0 +1,50 @@ +# Dockerfile generated by constructor {{ constructor_version }} + +########################################################## +# Stage 1: Run .sh installer +########################################################## + +FROM {{ base_image }} AS builder + +ARG PREFIX={{ default_prefix }} + +COPY {{ installer_filename }} /tmp/installer.sh + +RUN sh /tmp/installer.sh -b -p "${PREFIX}" && \ + rm -rf "${PREFIX}/uninstall.sh" && \ + rm -rf "${PREFIX}/_conda" || true && \ + find "${PREFIX}/lib" -name 'lib*.a' -delete && \ + find "${PREFIX}/lib" -name 'lib*.js.map' -delete && \ + find "${PREFIX}" -name '__pycache__' -type d -exec rm -rf {} + 2>/dev/null || true && \ + {% if not keep_pkgs %}rm -rf "${PREFIX}/pkgs" && {{ clean_cmd }} && \ + {% endif %}find "${PREFIX}" -follow -type f -name '*.a' -delete + +########################################################## +# Stage 2: Final image +########################################################## + +FROM {{ base_image }} + +ARG PREFIX={{ default_prefix }} + +LABEL org.opencontainers.image.title="{{ name }}" +LABEL org.opencontainers.image.version="{{ version }}" +{%- if labels %} +{%- for k, v in labels.items() %} +LABEL {{ k }}="{{ v }}" +{%- endfor %} +{%- endif %} + +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 +ENV PATH="${PREFIX}/bin:${PATH}" + +COPY --from=builder ${PREFIX} ${PREFIX} +{% if register_envs %} +COPY --from=builder /root/.conda /root/.conda +{% endif %} + +RUN echo 'export PATH=$(sed -e "s,:\?{{ default_prefix }}/bin:,," <<< "${PATH}")' >> ~/.bashrc && \ + {{ init_cmd }} init --all + +CMD [ "/bin/bash" ] diff --git a/constructor/main.py b/constructor/main.py index 5a8ce9af7..69da8cdc9 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -40,10 +40,30 @@ def get_installer_type(info: dict): osname, unused_arch = info["_platform"].split("-") - os_allowed = {"linux": ("sh",), "osx": ("sh", "pkg"), "win": ("exe",)} - all_allowed = set(sum(os_allowed.values(), ("all",))) + os_allowed = { + "linux": ("sh",), + "osx": ( + "sh", + "pkg", + ), + "win": ("exe",), + } + all_allowed = set(sum(os_allowed.values(), ("all", "docker"))) itype = info.get("installer_type") + docker_build = info.get("docker_build", False) + + if docker_build and osname == "win": + sys.exit( + "Error: 'docker_build' is not supported on Windows. " + "Run the build on Linux or macOS instead." + ) + if docker_build and itype in ("pkg", "exe"): + sys.exit( + "Error: 'docker_build' is not compatible with installer_type 'pkg' or 'exe'. " + "Use installer_type: 'sh', 'docker', or omit installer_type." + ) + if not itype: return os_allowed[osname][:1] elif itype == "all": @@ -51,6 +71,8 @@ def get_installer_type(info: dict): elif itype not in all_allowed: all_allowed = ", ".join(sorted(all_allowed)) sys.exit("Error: invalid installer type '%s'; allowed: %s" % (itype, all_allowed)) + elif itype == "docker" or docker_build: + return ("sh", "docker") elif itype not in os_allowed[osname]: os_allowed = ", ".join(sorted(os_allowed[osname])) sys.exit( @@ -202,6 +224,12 @@ def main_build( info["_debug"] = debug itypes = get_installer_type(info) + if "docker" in itypes and not info.get("docker_base_image"): + sys.exit( + "Error: docker_base_image is required when building Docker output. " + "Please specify a base image using the 'docker_base_image' key in construct.yaml." + ) + if platform != cc_platform and "pkg" in itypes and not cc_platform.startswith("osx-"): sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform) @@ -399,12 +427,27 @@ def main_build( from .winexe import create as winexe_create create = winexe_create + elif itype == "docker": + from .docker_build import create as docker_create + + create = docker_create info["installer_type"] = itype - info["_outpath"] = abspath(join(output_dir, get_output_filename(info))) + if itype != "docker": + info["_outpath"] = abspath(join(output_dir, get_output_filename(info))) + else: + info["_outpath"] = abspath(join(output_dir, get_output_filename(info))).replace( + ".docker", ".sh" + ) create(info, verbose=verbose) if len(itypes) > 1: info_dicts.append(info.copy()) - logger.info("Successfully created '%(_outpath)s'.", info) + if itype == "docker": + logger.info( + "Docker output complete. Docker directory: '%s'", + Path(info["_output_dir"]) / "docker", + ) + else: + logger.info("Successfully created '%(_outpath)s'.", info) # Merge info files for each installer type if len(itypes) > 1: diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index 17a71302f..ace5183bf 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -235,6 +235,7 @@ The type of the installer being created. Possible values are: - `sh`: shell-based installer for Linux or macOS - `pkg`: macOS GUI installer built with Apple's `pkgbuild` - `exe`: Windows GUI installer built with NSIS +- `docker`: generates a Dockerfile that replicates the installation environment (Does not build image. See `docker_build` option below.) The default type is `sh` on Linux and macOS, and `exe` on Windows. A special value of `all` builds _both_ `sh` and `pkg` installers on macOS, as well @@ -679,6 +680,29 @@ freeze_base: message: "This base environment is frozen and cannot be modified." ``` +### `docker_base_image` + +Base image to use for docker builds when `installer_type` includes `docker` or `docker_build` is True. +Should be a specific image reference. For reproducibility, please specify a SHA256 digest. +For example: `debian:13.4-slim@sha256:abc123...`. If the digest is not provided, a warning will be shown. + +### `docker_tag` + +Tag to use for the docker image. +If not provided, it will default to `:`. + +### `docker_labels` + +Additional labels to add to the built docker image. +The labels `org.opencontainers.image.title` and `org.opencontainers.image.version` are +set automatically from `name` and `version`. + +### `docker_build` + +If `True`, builds a docker image using the Dockerfile generated by constructor and saves it as a portable tarball. +``--.tar`` will be created in the output docker directory. +Requires `docker_base_image` to be specified. + ## Available selectors - `aarch64` diff --git a/examples/docker_build/construct.yaml b/examples/docker_build/construct.yaml new file mode 100644 index 000000000..19d5c08a9 --- /dev/null +++ b/examples/docker_build/construct.yaml @@ -0,0 +1,26 @@ +name: test_docker +version: 1.0.0 + +channels: + - conda-forge + +specs: + - python + - numpy + - conda + +installer_type: docker +# Generates Dockerfile and stages the installer. + +docker_build: true +# Additionally builds a portable Docker image from the generated Dockerfile. + +docker_base_image: "debian:13.4-slim@sha256:cedb1ef40439206b673ee8b33a46a03a0c9fa90bf3732f54704f99cb061d2c5a" + +keep_pkgs: true + +register_envs: true + +docker_labels: + maintainer: "jaidarice" + description: "Test Docker image built with constructor." diff --git a/news/1219-docker-implementation b/news/1219-docker-implementation new file mode 100644 index 000000000..b3dbbea7f --- /dev/null +++ b/news/1219-docker-implementation @@ -0,0 +1,20 @@ +### Enhancements + +* Add `installer_type: docker` support to generate a Dockerfile from a constructor build to be used as-is or modified. (#1219) +* Add `docker_build: true` support to build a portable Docker image. (#1219) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_examples.py b/tests/test_examples.py index 4ec5c9a64..b79a0b333 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1554,3 +1554,37 @@ def test_frozen_environment(tmp_path, request, has_conflict): s in c.value.stderr for s in ("RuntimeError", "freeze_base / freeze_env", "extra_files", "base") ) + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="Unix only") +@pytest.mark.skipif(not shutil.which("docker"), reason="Docker not available") +def test_docker_build(tmp_path): + input_path = _example_path("docker_build") + output_path = tmp_path / "output" + docker_dir = output_path / "installer" / "docker" + + yaml = YAML() + with open(input_path / "construct.yaml") as f: + config = yaml.load(f) + image_name = f"{config['name'].lower()}:{config['version'].split('-')[0]}" + + try: + for installer, _ in create_installer(input_path, output_path): + assert (docker_dir / "Dockerfile").exists() + assert (docker_dir / installer.name).exists() + + tarballs = list(docker_dir.glob("*.tar")) + assert tarballs, "No Docker image tarball found in docker dir" + subprocess.run(["docker", "load", "-i", str(tarballs[0])], check=True) + + result = subprocess.run( + ["docker", "run", "--rm", image_name, "conda", "--version"], + capture_output=True, + text=True, + check=True, + ) + + assert "conda" in result.stdout + + finally: + subprocess.run(["docker", "rmi", image_name], check=False)