diff --git a/src/rapids_dependency_file_generator/_constants.py b/src/rapids_dependency_file_generator/_constants.py index 2a3490ab..8d120fba 100644 --- a/src/rapids_dependency_file_generator/_constants.py +++ b/src/rapids_dependency_file_generator/_constants.py @@ -9,4 +9,5 @@ default_conda_dir = "conda/environments" default_requirements_dir = "python" default_pyproject_dir = "python" + default_dependency_file_path = "dependencies.yaml" diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py index c01c63e2..214b57f1 100644 --- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py +++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py @@ -5,6 +5,7 @@ import typing from collections.abc import Generator from dataclasses import dataclass +from pathlib import Path import tomlkit import yaml @@ -213,6 +214,54 @@ def make_dependency_file( return file_contents +def make_conda_build_config_file( + *, + config_file: os.PathLike, + output_dir: os.PathLike, + matrix: dict[str, list[str]], +) -> str: + """Generate the contents of a conda_build_config.yaml file. + + This function aggregates matrix values from the dependencies.yaml file and + formats them in the conda-build config format. + + Parameters + ---------- + config_file : PathLike + The full path to the dependencies.yaml file. + output_dir : PathLike + The path to the directory where the conda_build_config.yaml file will be written. + matrix : dict[str, list[str]] + The matrix of specific parameters to use when generating. + + Returns + ------- + str + The contents of the conda_build_config.yaml file. + """ + relative_path_to_config_file = os.path.relpath(config_file, output_dir) + file_contents = textwrap.dedent( + f"""\ + {HEADER} + # To make changes, edit {relative_path_to_config_file} and run `{cli_name}`. + # This file contains matrix values aggregated from dependencies.yaml + """ + ) + + # Convert matrix values to conda-build config format + conda_build_config = {} + for key, values in matrix.items(): + # Rename "py" key to "python" for conda-build compatibility + if key == "py": + conda_build_config["python"] = values + else: + conda_build_config[key] = values + + file_contents += yaml.dump(conda_build_config, default_flow_style=False) + + return file_contents + + def get_filename(file_type: _config.Output, file_key: str, matrix_combo: dict[str, str]): """Get the name of the file to which to write a generated dependency set. @@ -256,6 +305,7 @@ def get_filename(file_type: _config.Output, file_key: str, matrix_combo: dict[st # need to have that exact name and are never prefixed. file_name_prefix = "pyproject" suffix = "" + filename = "_".join(filter(None, (file_type_prefix, file_name_prefix, suffix))).replace(".", "") return filename + file_ext @@ -289,6 +339,7 @@ def get_output_dir(*, file_type: _config.Output, config_file_path: os.PathLike, path.append(file_config.requirements_dir) elif file_type == _config.Output.PYPROJECT: path.append(file_config.pyproject_dir) + return os.path.join(*path) @@ -417,6 +468,25 @@ def make_dependency_files( if _config.Output.PYPROJECT in file_types_to_generate and len(calculated_grid) > 1: raise ValueError("Pyproject outputs can't have more than one matrix output") for file_type in file_types_to_generate: + # Generate conda_build_config.yaml if this is a conda output type + if file_type == _config.Output.CONDA and not to_stdout: + conda_build_config_name = "conda_build_config.yaml" + output_dir = get_output_dir( + file_type=file_type, + config_file_path=parsed_config.path, + file_config=file_config, + ) + conda_build_config_contents = make_conda_build_config_file( + config_file=parsed_config.path, + output_dir=output_dir, + matrix=file_matrix, + ) + + os.makedirs(output_dir, exist_ok=True) + conda_build_config_path = Path(output_dir) / conda_build_config_name + with open(conda_build_config_path, "w") as f: + f.write(conda_build_config_contents) + for matrix_combo in calculated_grid: dependencies = [] diff --git a/tests/examples/conda-build-config/dependencies.yaml b/tests/examples/conda-build-config/dependencies.yaml new file mode 100644 index 00000000..f83a4618 --- /dev/null +++ b/tests/examples/conda-build-config/dependencies.yaml @@ -0,0 +1,41 @@ +files: + build: + output: [conda] + conda_dir: output/actual + matrix: + cuda: ["11.5", "11.6"] + arch: [x86_64, arm64] + py: ["3.9", "3.10"] + includes: + - build +channels: + - rapidsai + - conda-forge +dependencies: + build: + common: + - output_types: [conda] + packages: + - clang-tools=11.1.0 + - spdlog>=1.8.5,<1.9 + specific: + - output_types: [conda] + matrices: + - matrix: + cuda: "11.5" + packages: + - cudatoolkit=11.5 + - matrix: + cuda: "11.6" + packages: + - cudatoolkit=11.6 + - output_types: [conda] + matrices: + - matrix: + py: "3.9" + packages: + - python=3.9 + - matrix: + py: "3.10" + packages: + - python=3.10 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-arm64_py-310.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-arm64_py-310.yaml new file mode 100644 index 00000000..d98e93e3 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-arm64_py-310.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.5 +- python=3.10 +- spdlog>=1.8.5,<1.9 +name: build_cuda-115_arch-arm64_py-310 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-arm64_py-39.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-arm64_py-39.yaml new file mode 100644 index 00000000..9caa2412 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-arm64_py-39.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.5 +- python=3.9 +- spdlog>=1.8.5,<1.9 +name: build_cuda-115_arch-arm64_py-39 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-x86_64_py-310.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-x86_64_py-310.yaml new file mode 100644 index 00000000..edfc3290 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-x86_64_py-310.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.5 +- python=3.10 +- spdlog>=1.8.5,<1.9 +name: build_cuda-115_arch-x86_64_py-310 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-x86_64_py-39.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-x86_64_py-39.yaml new file mode 100644 index 00000000..aa889b12 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-115_arch-x86_64_py-39.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.5 +- python=3.9 +- spdlog>=1.8.5,<1.9 +name: build_cuda-115_arch-x86_64_py-39 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-arm64_py-310.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-arm64_py-310.yaml new file mode 100644 index 00000000..aefd2226 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-arm64_py-310.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.6 +- python=3.10 +- spdlog>=1.8.5,<1.9 +name: build_cuda-116_arch-arm64_py-310 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-arm64_py-39.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-arm64_py-39.yaml new file mode 100644 index 00000000..ddda31ed --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-arm64_py-39.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.6 +- python=3.9 +- spdlog>=1.8.5,<1.9 +name: build_cuda-116_arch-arm64_py-39 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-x86_64_py-310.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-x86_64_py-310.yaml new file mode 100644 index 00000000..7da49279 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-x86_64_py-310.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.6 +- python=3.10 +- spdlog>=1.8.5,<1.9 +name: build_cuda-116_arch-x86_64_py-310 diff --git a/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-x86_64_py-39.yaml b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-x86_64_py-39.yaml new file mode 100644 index 00000000..5475e020 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/build_cuda-116_arch-x86_64_py-39.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- clang-tools=11.1.0 +- cudatoolkit=11.6 +- python=3.9 +- spdlog>=1.8.5,<1.9 +name: build_cuda-116_arch-x86_64_py-39 diff --git a/tests/examples/conda-build-config/output/expected/conda_build_config.yaml b/tests/examples/conda-build-config/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..f0330ff1 --- /dev/null +++ b/tests/examples/conda-build-config/output/expected/conda_build_config.yaml @@ -0,0 +1,12 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +arch: +- x86_64 +- arm64 +cuda: +- '11.5' +- '11.6' +python: +- '3.9' +- '3.10' diff --git a/tests/examples/conda-minimal/output/expected/conda_build_config.yaml b/tests/examples/conda-minimal/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..9af565a9 --- /dev/null +++ b/tests/examples/conda-minimal/output/expected/conda_build_config.yaml @@ -0,0 +1,8 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +arch: +- x86_64 +cuda: +- '11.5' +- '11.6' diff --git a/tests/examples/integration/output/expected/conda_build_config.yaml b/tests/examples/integration/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..1ff60de8 --- /dev/null +++ b/tests/examples/integration/output/expected/conda_build_config.yaml @@ -0,0 +1,6 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +cuda: +- '11.5' +- '11.6' diff --git a/tests/examples/matrix-glob/output/expected/conda_build_config.yaml b/tests/examples/matrix-glob/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..39608f3a --- /dev/null +++ b/tests/examples/matrix-glob/output/expected/conda_build_config.yaml @@ -0,0 +1,7 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +cuda: +- '10.0' +- '11.8' +- '12.0' diff --git a/tests/examples/matrix-null-item/output/expected/conda_build_config.yaml b/tests/examples/matrix-null-item/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..02b9a241 --- /dev/null +++ b/tests/examples/matrix-null-item/output/expected/conda_build_config.yaml @@ -0,0 +1,11 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +arch: +- x86_64 +- arm64 +cuda: +- '11.5' +python: +- '3.8' +- null diff --git a/tests/examples/matrix/output/expected/conda_build_config.yaml b/tests/examples/matrix/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..5db1fa73 --- /dev/null +++ b/tests/examples/matrix/output/expected/conda_build_config.yaml @@ -0,0 +1,12 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +arch: +- x86_64 +- arm64 +cuda: +- '11.5' +- '11.6' +python: +- '3.8' +- '3.9' diff --git a/tests/examples/no-matrix/output/expected/conda_build_config.yaml b/tests/examples/no-matrix/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..b8929cfd --- /dev/null +++ b/tests/examples/no-matrix/output/expected/conda_build_config.yaml @@ -0,0 +1,4 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +{} diff --git a/tests/examples/no-specific-match/conda/environments/conda_build_config.yaml b/tests/examples/no-specific-match/conda/environments/conda_build_config.yaml new file mode 100644 index 00000000..c9a8fec3 --- /dev/null +++ b/tests/examples/no-specific-match/conda/environments/conda_build_config.yaml @@ -0,0 +1,5 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +cuda: +- '11.8' diff --git a/tests/examples/prepend-channels/output/expected/conda_build_config.yaml b/tests/examples/prepend-channels/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..9af565a9 --- /dev/null +++ b/tests/examples/prepend-channels/output/expected/conda_build_config.yaml @@ -0,0 +1,8 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +arch: +- x86_64 +cuda: +- '11.5' +- '11.6' diff --git a/tests/examples/specific-fallback-first/output/expected/conda_build_config.yaml b/tests/examples/specific-fallback-first/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..2690d466 --- /dev/null +++ b/tests/examples/specific-fallback-first/output/expected/conda_build_config.yaml @@ -0,0 +1,6 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +cuda: +- '11.5' +- '11.8' diff --git a/tests/examples/specific-fallback-multiple-matches/output/expected/conda_build_config.yaml b/tests/examples/specific-fallback-multiple-matches/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..2690d466 --- /dev/null +++ b/tests/examples/specific-fallback-multiple-matches/output/expected/conda_build_config.yaml @@ -0,0 +1,6 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +cuda: +- '11.5' +- '11.8' diff --git a/tests/examples/specific-fallback/output/expected/conda_build_config.yaml b/tests/examples/specific-fallback/output/expected/conda_build_config.yaml new file mode 100644 index 00000000..2690d466 --- /dev/null +++ b/tests/examples/specific-fallback/output/expected/conda_build_config.yaml @@ -0,0 +1,6 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +# This file contains matrix values aggregated from dependencies.yaml +cuda: +- '11.5' +- '11.8' diff --git a/tests/test_conda_build_config.py b/tests/test_conda_build_config.py new file mode 100644 index 00000000..f2bb765a --- /dev/null +++ b/tests/test_conda_build_config.py @@ -0,0 +1,215 @@ +"""Test that conda output type also generates conda_build_config.yaml.""" +import tempfile +from pathlib import Path + +import yaml + +from rapids_dependency_file_generator._config import Output, load_config_from_file +from rapids_dependency_file_generator._rapids_dependency_file_generator import ( + make_dependency_files, +) + + +def test_conda_build_config_output(): + """Test that conda output type also generates conda_build_config.yaml files.""" + # Create a temporary directory for the test + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create a test dependencies.yaml file + test_config = { + "files": { + "build": { + "output": ["conda"], + "conda_dir": "output", + "matrix": { + "cuda": ["11.5", "11.6"], + "arch": ["x86_64", "arm64"], + "py": ["3.8", "3.9"] + }, + "includes": ["build"] + } + }, + "channels": ["rapidsai", "conda-forge"], + "dependencies": { + "build": { + "common": [ + { + "output_types": ["conda"], + "packages": ["clang-tools=11.1.0"] + } + ] + } + } + } + + config_file = temp_path / "dependencies.yaml" + with open(config_file, "w") as f: + yaml.dump(test_config, f) + + # Load the config + parsed_config = load_config_from_file(config_file) + + # Generate the conda files (including conda_build_config.yaml) + make_dependency_files( + parsed_config=parsed_config, + file_keys=["build"], + output={Output.CONDA}, + matrix=None, + prepend_channels=[], + to_stdout=False, + ) + + # Check that the file was created + output_file = temp_path / "output" / "conda_build_config.yaml" + assert output_file.exists() + + # Read and verify the contents + with open(output_file) as f: + content = f.read() + + # Check that the header is present + assert "# This file is generated by `rapids-dependency-file-generator`" in content + assert "# This file contains matrix values aggregated from dependencies.yaml" in content + + # Parse the YAML content + with open(output_file) as f: + parsed_content = yaml.safe_load(f) + + # Verify the matrix values are correct + assert parsed_content["cuda"] == ["11.5", "11.6"] + assert parsed_content["arch"] == ["x86_64", "arm64"] + assert parsed_content["python"] == ["3.8", "3.9"] + + +def test_conda_build_config_with_custom_matrix(): + """Test that conda_build_config.yaml works with custom matrix values.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create a test dependencies.yaml file + test_config = { + "files": { + "build": { + "output": ["conda"], + "conda_dir": "output", + "matrix": { + "cuda": ["11.5", "11.6", "11.7"], + "arch": ["x86_64"] + }, + "includes": ["build"] + } + }, + "channels": ["rapidsai"], + "dependencies": { + "build": { + "common": [ + { + "output_types": ["conda"], + "packages": ["clang-tools=11.1.0"] + } + ] + } + } + } + + config_file = temp_path / "dependencies.yaml" + with open(config_file, "w") as f: + yaml.dump(test_config, f) + + # Load the config + parsed_config = load_config_from_file(config_file) + + # Generate with a custom matrix + custom_matrix = { + "cuda": ["11.8"], + "arch": ["arm64"] + } + + make_dependency_files( + parsed_config=parsed_config, + file_keys=["build"], + output={Output.CONDA}, + matrix=custom_matrix, + prepend_channels=[], + to_stdout=False, + ) + + # Check that the file was created + output_file = temp_path / "output" / "conda_build_config.yaml" + assert output_file.exists() + + # Parse the YAML content + with open(output_file) as f: + parsed_content = yaml.safe_load(f) + + # Verify the custom matrix values are used + assert parsed_content["cuda"] == ["11.8"] + assert parsed_content["arch"] == ["arm64"] + + +def test_conda_build_config_stdout(): + """Test that conda_build_config output works with stdout.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create a test dependencies.yaml file + test_config = { + "files": { + "build": { + "output": ["conda"], + "conda_dir": "output", + "matrix": { + "cuda": ["11.5"], + "arch": ["x86_64"] + }, + "includes": ["build"] + } + }, + "channels": ["rapidsai"], + "dependencies": { + "build": { + "common": [ + { + "output_types": ["conda"], + "packages": ["clang-tools=11.1.0"] + } + ] + } + } + } + + config_file = temp_path / "dependencies.yaml" + with open(config_file, "w") as f: + yaml.dump(test_config, f) + + # Load the config + parsed_config = load_config_from_file(config_file) + + # Capture stdout output + import io + import sys + + captured_output = io.StringIO() + original_stdout = sys.stdout + sys.stdout = captured_output + + try: + make_dependency_files( + parsed_config=parsed_config, + file_keys=["build"], + output={Output.CONDA}, + matrix=None, + prepend_channels=[], + to_stdout=True, + ) + finally: + sys.stdout = original_stdout + + output = captured_output.getvalue() + + # Check that the output contains the expected content (conda environment file) + assert "# This file is generated by `rapids-dependency-file-generator`" in output + assert "channels:" in output + assert "dependencies:" in output + assert "clang-tools=11.1.0" in output