diff --git a/lib/ramble/ramble/cmd/workspace.py b/lib/ramble/ramble/cmd/workspace.py index 8bd36e1f3f..155c8c4ca4 100644 --- a/lib/ramble/ramble/cmd/workspace.py +++ b/lib/ramble/ramble/cmd/workspace.py @@ -879,7 +879,7 @@ def workspace_info(args): color.cprint(f" {template}") # Print workspace variables information - workspace_vars = ws.get_workspace_vars() + workspace_vars = ramble.config.get(namespace.variables) ws.software_environments = ramble.software_environments.SoftwareEnvironments(ws) software_environments = ws.software_environments @@ -1140,7 +1140,7 @@ def workspace_info(args): if not all_utilities: color.cprint(" None") else: - ws_utilities = ws.get_workspace_utilities() or {} + ws_utilities = ramble.config.get(namespace.utilities) or {} for utility_name, confs in sorted(all_utilities.items()): color.cprint(color.nested_1(f" {utility_name}:")) diff --git a/lib/ramble/ramble/experiment_set.py b/lib/ramble/ramble/experiment_set.py index af059cc521..15efce6784 100644 --- a/lib/ramble/ramble/experiment_set.py +++ b/lib/ramble/ramble/experiment_set.py @@ -14,6 +14,7 @@ from functools import partial from typing import Set +import ramble.config import ramble.context import ramble.error import ramble.expander @@ -63,15 +64,17 @@ def __init__(self, workspace): # Set all workspace variables as base variables. workspace_context = ramble.context.Context() workspace_context.context_name = workspace.name - workspace_context.variables = workspace.get_workspace_vars() - workspace_context.env_variables = workspace.get_workspace_env_vars() - workspace_context.formatted_executables = workspace.get_workspace_formatted_executables() - workspace_context.internals = workspace.get_workspace_internals() - workspace_context.modifiers = workspace.get_workspace_modifiers() - workspace_context.zips = workspace.get_workspace_zips() - workspace_context.variants = workspace.get_workspace_variants() - workspace_context.success_criteria = workspace.get_workspace_success_criteria() - workspace_context.tables = workspace.get_workspace_tables() + workspace_context.variables = ramble.config.get(namespace.variables) + workspace_context.env_variables = ramble.config.get(namespace.env_var) + workspace_context.formatted_executables = ramble.config.get( + namespace.formatted_executables + ) + workspace_context.internals = ramble.config.get(namespace.internals) + workspace_context.modifiers = ramble.config.get(namespace.modifiers) + workspace_context.zips = ramble.config.get(namespace.zips) + workspace_context.variants = ramble.config.get(namespace.variants) + workspace_context.success_criteria = ramble.config.get(namespace.success) + workspace_context.tables = ramble.config.get(namespace.tables) try: self.keywords.check_reserved_keys(workspace_context.variables) diff --git a/lib/ramble/ramble/pipeline.py b/lib/ramble/ramble/pipeline.py index bdd8335bdc..7fe0186e32 100644 --- a/lib/ramble/ramble/pipeline.py +++ b/lib/ramble/ramble/pipeline.py @@ -27,6 +27,7 @@ import ramble.util.hashing import ramble.util.path import ramble.workspace +from ramble.namespace import namespace from ramble.util import json_util from ramble.util.colors import cprint from ramble.util.file_util import create_symlink @@ -783,7 +784,7 @@ def __init__( ): super().__init__(workspace, filters) - workspace_expander = ramble.expander.Expander(workspace.get_workspace_vars(), None) + workspace_expander = ramble.expander.Expander(ramble.config.get(namespace.variables), None) self.action_string = "Pushing deployment of" self.create_tar = create_tar diff --git a/lib/ramble/ramble/software_environments.py b/lib/ramble/ramble/software_environments.py index 4fa7d35ef2..0106039bab 100644 --- a/lib/ramble/ramble/software_environments.py +++ b/lib/ramble/ramble/software_environments.py @@ -6,9 +6,11 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import copy from collections import defaultdict from typing import DefaultDict, Dict, List, Set +import ramble.config import ramble.error import ramble.util.colors as color from ramble.expander import Expander @@ -714,7 +716,7 @@ def __init__(self, workspace): """ self._workspace = workspace - self._software_dict = workspace.get_software_dict().copy() + self._software_dict = copy.deepcopy(ramble.config.get(namespace.software)) self._environment_templates = {} self._external_env_templates = {} self._package_templates = {} diff --git a/lib/ramble/ramble/test/cmd/workspace.py b/lib/ramble/ramble/test/cmd/workspace.py index 0729b5d0b0..ddc2b7c86a 100644 --- a/lib/ramble/ramble/test/cmd/workspace.py +++ b/lib/ramble/ramble/test/cmd/workspace.py @@ -1648,21 +1648,21 @@ def write_config(ws_path, config): with fs.working_dir(config_path): write_config(ws_path, test_config) - with ramble.workspace.Workspace(ws_path) as ws: - software_dict = ws.get_software_dict() + with ramble.workspace.Workspace(ws_path): + software_dict = ramble.config.get(namespace.software) print(f"software_dict before = {software_dict}") workspace("concretize", global_args=workspace_flags) - with ramble.workspace.Workspace(ws_path) as ws: - software_dict = ws.get_software_dict() + with ramble.workspace.Workspace(ws_path): + software_dict = ramble.config.get(namespace.software) assert namespace.environments in software_dict write_config(ws_path, test_config) workspace("concretize", global_args=workspace_flags) - with ramble.workspace.Workspace(ws_path) as ws: - software_dict = ws.get_software_dict() + with ramble.workspace.Workspace(ws_path): + software_dict = ramble.config.get(namespace.software) assert namespace.environments in software_dict diff --git a/lib/ramble/ramble/test/end_to_end/concretize_with_different_package_managers.py b/lib/ramble/ramble/test/end_to_end/concretize_with_different_package_managers.py index 5896cd9179..54db50f17d 100644 --- a/lib/ramble/ramble/test/end_to_end/concretize_with_different_package_managers.py +++ b/lib/ramble/ramble/test/end_to_end/concretize_with_different_package_managers.py @@ -10,8 +10,10 @@ import pytest +import ramble.config import ramble.workspace from ramble.main import RambleCommand +from ramble.namespace import namespace # everything here uses the mock_workspace_path pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_workspace_path") @@ -66,7 +68,7 @@ def test_concretize_with_different_package_managers( workspace("concretize", global_args=["-w", workspace_name]) - sw_dict = ws1.get_software_dict() + sw_dict = ramble.config.get(namespace.software) assert "wrfv4" in sw_dict["environments"] assert "wrfv4" in sw_dict["packages"] diff --git a/lib/ramble/ramble/test/software_environment.py b/lib/ramble/ramble/test/software_environment.py index 881333da4b..2bfcd43d21 100644 --- a/lib/ramble/ramble/test/software_environment.py +++ b/lib/ramble/ramble/test/software_environment.py @@ -8,10 +8,12 @@ import pytest +import ramble.config import ramble.expander import ramble.software_environments import ramble.workspace from ramble.main import RambleCommand +from ramble.namespace import namespace pytestmark = pytest.mark.usefixtures( "mutable_config", @@ -33,7 +35,7 @@ def test_basic_software_environment(request, mutable_mock_workspace_path): assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic"] = {"pkg_spec": "basic@1.1"} @@ -66,7 +68,7 @@ def test_software_environments_no_packages(request, mutable_mock_workspace_path) assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["environments"] = {"basic-{env_test}": {"packages": [""]}} @@ -94,7 +96,7 @@ def test_software_environments_no_rendered_packages(request, mutable_mock_worksp assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["environments"] = {"basic-{env_test}": {"packages": ["{var_pkg_name}"]}} @@ -120,7 +122,7 @@ def test_template_software_environments(request, mutable_mock_workspace_path): assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic-{pkg_test}"] = {"pkg_spec": "basic@1.1"} @@ -156,7 +158,7 @@ def test_multi_template_software_environments(request, mutable_mock_workspace_pa assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic1-{pkg_test}"] = {"pkg_spec": "basic@1.1"} @@ -210,7 +212,7 @@ def test_undefined_package_errors(request, mutable_mock_workspace_path): assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic-{pkg_test}"] = {"pkg_spec": "basic@{pkg_ver}"} @@ -244,7 +246,7 @@ def test_invalid_packages_error(request, mutable_mock_workspace_path): assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic-{pkg_test}"] = {"pkg_spec": "basic@{pkg_ver}"} @@ -290,7 +292,7 @@ def test_invalid_environment_error(request, mutable_mock_workspace_path): assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic1-{pkg_test}"] = {"pkg_spec": "basic@1.1"} @@ -335,7 +337,7 @@ def test_undefined_compiler_errors(request, mutable_mock_workspace_path): assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic"] = {"pkg_spec": "basic@1.1", "compiler": "foo_comp"} @@ -364,7 +366,7 @@ def test_compiler_in_environment_warns(request, mutable_mock_workspace_path, cap assert ws_name in workspace("list") with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["test_comp"] = {"pkg_spec": "comp@2.1"} @@ -391,7 +393,7 @@ def test_is_used_property(request, mutable_mock_workspace_path): workspace("create", ws_name) with ramble.workspace.read(ws_name) as ws: - software_dict = ws.get_software_dict() + software_dict = ramble.config.get(namespace.software) software_dict["packages"] = {} software_dict["packages"]["basic"] = {"pkg_spec": "basic@1.1"} software_dict["environments"] = {"basic": {"packages": ["basic"]}} diff --git a/lib/ramble/ramble/test/workspace.py b/lib/ramble/ramble/test/workspace.py index 67008c77e6..0aae8fa6e4 100644 --- a/lib/ramble/ramble/test/workspace.py +++ b/lib/ramble/ramble/test/workspace.py @@ -12,7 +12,8 @@ import pytest -import ramble +import ramble.config +from ramble.namespace import namespace from ramble.workspace import TEMPLATE_EXTENSION, workspace # everything here uses the mock_workspace_path @@ -182,7 +183,7 @@ def test_add_experiments_with_zips(make_workspace_from_config): zips=["my_zip=[var1,var2]"], ) ws._re_read() - app_config = ws.get_applications() + app_config = ramble.config.get(namespace.application) exp_config = app_config["basic"]["workloads"]["test_wl"]["experiments"]["test_exp"] assert "zips" in exp_config assert exp_config["zips"]["my_zip"] == ["var1", "var2"] diff --git a/lib/ramble/ramble/workspace/workspace.py b/lib/ramble/ramble/workspace/workspace.py index 56a764ee9b..784cdb2700 100644 --- a/lib/ramble/ramble/workspace/workspace.py +++ b/lib/ramble/ramble/workspace/workspace.py @@ -904,7 +904,7 @@ def all_applications(self): logger.debug(f" With ws dict: {ws_dict}") # Iterate over applications in ramble.yaml first - app_dict = ramble.config.config.get_config(namespace.application) + app_dict = copy.deepcopy(ramble.config.get(namespace.application)) for application, contents in app_dict.items(): app_name, _, maybe_version = application.partition("@") @@ -1007,7 +1007,7 @@ def manage_environments( if package_list and external_path is not None: logger.die("Can only manage environments with one of package_list or external_path") - software_dict = self.get_software_dict().copy() + software_dict = copy.deepcopy(ramble.config.get(namespace.software)) if namespace.environments in software_dict: environments = software_dict[namespace.environments] @@ -1082,7 +1082,7 @@ def manage_packages( overwrite (bool): Whether colliding definitions should be overwritten """ - software_dict = self.get_software_dict().copy() + software_dict = copy.deepcopy(ramble.config.get(namespace.software)) if namespace.packages in software_dict: packages = software_dict[namespace.packages] @@ -1277,8 +1277,8 @@ def process_definitions(definitions, def_type="variable"): edited = False - workspace_vars = self.get_workspace_vars() - apps_dict = self.get_applications().copy() + workspace_vars = ramble.config.get(namespace.variables) + apps_dict = copy.deepcopy(ramble.config.get(namespace.application)) app_inst = ramble.repository.get(application) @@ -1478,7 +1478,7 @@ def concretize(self, force=False, quiet=False): """ - full_software_dict = self.get_software_dict() + full_software_dict = copy.deepcopy(ramble.config.get(namespace.software)) if ( namespace.packages not in full_software_dict @@ -2019,8 +2019,8 @@ def _remove_scoped_variables(scope_name: str, used_variables: Set): def simplify_software(self): # First drop unused experiment templates from app dict so environments aren't rendered - app_dict = ramble.config.config.get_config( - namespace.application, scope=self.ws_file_config_scope_name() + app_dict = copy.deepcopy( + ramble.config.get(namespace.application, scope=self.ws_file_config_scope_name()) ) # Build experiment sets to determine which templates never get used @@ -2063,8 +2063,8 @@ def simplify_software(self): package_dict = None environments_dict = None - software_dict = ramble.config.config.get_config( - namespace.software, scope=self.ws_file_config_scope_name() + software_dict = copy.deepcopy( + ramble.config.get(namespace.software, scope=self.ws_file_config_scope_name()) ) if namespace.packages in software_dict: @@ -2693,60 +2693,6 @@ def _get_workspace_dict(self): def _get_application_dict_config(self, key): return self.application_configs[key]["yaml"] if key in self.application_configs else None - def get_workspace_vars(self): - """Return a dict of workspace variables""" - return ramble.config.config.get_config(namespace.variables) - - def get_workspace_env_vars(self): - """Return a dict of workspace environment variables""" - return ramble.config.config.get_config(namespace.env_var) - - def get_workspace_formatted_executables(self): - """Return a dict of workspace formatted executables""" - return ramble.config.config.get_config(namespace.formatted_executables) - - def get_workspace_internals(self): - """Return a dict of workspace internals""" - return ramble.config.config.get_config(namespace.internals) - - def get_workspace_modifiers(self): - """Return a dict of workspace modifiers""" - return ramble.config.config.get_config(namespace.modifiers) - - def get_workspace_zips(self): - """Return a dict of workspace zips""" - return ramble.config.config.get_config(namespace.zips) - - def get_workspace_variants(self): - """Return a dict of workspace variants""" - return ramble.config.config.get_config(namespace.variants) - - def get_workspace_success_criteria(self): - """Return a dict of workspace success_criteria""" - return ramble.config.config.get_config(namespace.success) - - def get_software_dict(self): - """Return the software dictionary for this workspace""" - software_dict = ramble.config.config.get_config(namespace.software) - return software_dict - - def get_workspace_tables(self): - """Return a dict of workspace tables""" - return ramble.config.config.get_config(namespace.tables) - - def get_workspace_utilities(self): - """Return a dict of workspace utilities""" - return ramble.config.config.get_config(namespace.utilities) - - def get_applications(self): - """Get the dictionary of applications""" - logger.debug("Getting app dict.") - logger.debug(f" {self._get_workspace_dict()}") - workspace_dict = self._get_workspace_dict() - if namespace.application not in workspace_dict[namespace.ramble]: - workspace_dict[namespace.ramble][namespace.application] = syaml.syaml_dict() - return workspace_dict[namespace.ramble][namespace.application] - def read_transaction(self): """Get a read lock context manager for use in a `with` block.""" return lk.ReadTransaction(self.txlock, acquire=self._re_read) diff --git a/var/ramble/repos/builtin/workflow_managers/google-batch/test/google_batch.py b/var/ramble/repos/builtin/workflow_managers/google-batch/test/google_batch.py index edd179b399..0b7721fe1e 100644 --- a/var/ramble/repos/builtin/workflow_managers/google-batch/test/google_batch.py +++ b/var/ramble/repos/builtin/workflow_managers/google-batch/test/google_batch.py @@ -58,7 +58,7 @@ def test_google_batch_workflow_default(request): ws._re_read() # Remove batch submit definition - ws_vars = ws.get_workspace_vars() + ws_vars = ramble.config.get(namespace.variables) if "batch_submit" in ws_vars: del ws_vars["batch_submit"] ramble.config.config.update_config(