Resolve simtools-tests directory when using integration tests config files in applications.

dependency_versions.yml

@@ -22,7 +22,7 @@ corsika-interaction-tables:
   tag: v1.0.0
 model-database:
   name: CTAO-Simulation-Model
-  default-tag: v0.17.0
+  default-tag: v0.17.1
 simtools-tests:
   repository: gammasim/simtools-tests
   source-url: https://github.com/gammasim/simtools-tests.git

docs/changes/2473.feature.md

@@ -0,0 +1 @@
+Sort production grid tables in primary, array layout name, zenith, and model version.

docs/changes/2483.model.md

@@ -0,0 +1 @@
+Update default simulation models tag to `v0.17.1`.

src/simtools/constants.py

@@ -33,9 +33,29 @@
 # Path to resource files
 RESOURCE_PATH = files("simtools") / "resources"
 # Paths to test resources
-TEST_RESOURCES_ROOT = Path(
-    os.environ.get("SIMTOOLS_TEST_RESOURCES", "tests/unit_tests/resources")
-).expanduser()
+_DEFAULT_TEST_RESOURCES_ROOT = Path("tests/unit_tests/resources")
+
+
+def _configured_test_resources_root():
+    """Return the test-resource root configured through environment variables."""
+    configured_path = os.environ.get("SIMTOOLS_TEST_RESOURCES")
+    if configured_path:
+        return Path(configured_path).expanduser()
+
+    tests_path = os.environ.get("SIMTOOLS_TESTS_PATH")
+    tests_tag = os.environ.get("SIMTOOLS_TESTS_TAG") or os.environ.get("SIMTOOLS_TESTS_VERSION")
+    if tests_path and tests_tag:
+        return Path(tests_path).expanduser() / tests_tag / "integration_tests"
+
+    return None
+
+
+def get_test_resources_root():
+    """Return the active test-resource root."""
+    return _configured_test_resources_root() or TEST_RESOURCES_ROOT
+
+
+TEST_RESOURCES_ROOT = _configured_test_resources_root() or _DEFAULT_TEST_RESOURCES_ROOT
 TEST_RESOURCES_STATIC = str(TEST_RESOURCES_ROOT / "static")
 TEST_RESOURCES_GENERATED = str(TEST_RESOURCES_ROOT / "generated")
 TEST_RESOURCES_DOWNLOADED = str(TEST_RESOURCES_ROOT / "downloaded")

src/simtools/io/io_handler.py

@@ -20,15 +20,18 @@ def resolve_test_resource_paths(value, test_resources_path=None):
         Configuration value, mapping, or sequence to resolve.
     test_resources_path : str or pathlib.Path, optional
         Base directory containing the ``static``, ``generated``, and ``downloaded``
-        resource directories. Defaults to ``SIMTOOLS_TEST_RESOURCES`` or the unit-test
-        resource directory.
+        resource directories. Defaults to ``SIMTOOLS_TEST_RESOURCES`` or the versioned
+        ``simtools-tests`` integration-test resources selected by ``SIMTOOLS_TESTS_PATH`` and
+        ``SIMTOOLS_TESTS_TAG``.
 
     Returns
     -------
     object
         Configuration with absolute test-resource paths.
     """
-    base_path = Path(test_resources_path or constants.TEST_RESOURCES_ROOT).expanduser().resolve()
+    base_path = (
+        Path(test_resources_path or constants.get_test_resources_root()).expanduser().resolve()
+    )
     if isinstance(value, dict):
         return {
             key: resolve_test_resource_paths(item, test_resources_path=base_path)
@@ -64,7 +67,7 @@ def __init__(self):
         self.logger = logging.getLogger(__name__)
         self.output_path = {}
         self.model_path = None
-        self.test_resources_path = constants.TEST_RESOURCES_ROOT.resolve()
+        self.test_resources_path = constants.get_test_resources_root().resolve()
 
     def set_paths(self, output_path=None, model_path=None, output_path_label="default"):
         """

src/simtools/production_configuration/job_grid_io.py

@@ -281,6 +281,20 @@ def serialize_job_grid(job_rows, output_file, metadata=None):
     ]
     output_table = _build_output_table(output_rows, output_columns, metadata)
     output_table = _validate_job_grid_table(output_table)
+    output_table.sort(
+        [
+            "primary",
+            "model_version",
+            "array_layout_name",
+            "corsika_le_interaction",
+            "corsika_he_interaction",
+            "azimuth_angle",
+            "zenith_angle",
+            "energy_min",
+            "energy_max",
+            "run_number",
+        ]
+    )
     logger.info(f"Writing job grid with {len(job_rows)} rows to '{output_path}'.")
     output_table.write(output_path, format=_ECSV_FORMAT, overwrite=True)
 

tests/unit_tests/configuration/test_configurator.py

@@ -124,7 +124,11 @@ def test_config_from_file_rejects_inconsistent_unpreserved_by_version_key(
         config_builder._config_from_file(config_file)
 
 
-def test_config_from_file_resolves_test_resource_paths(tmp_test_directory):
+def test_config_from_file_resolves_test_resource_paths(tmp_test_directory, monkeypatch):
+    monkeypatch.delenv("SIMTOOLS_TEST_RESOURCES", raising=False)
+    monkeypatch.delenv("SIMTOOLS_TESTS_PATH", raising=False)
+    monkeypatch.delenv("SIMTOOLS_TESTS_TAG", raising=False)
+    monkeypatch.delenv("SIMTOOLS_TESTS_VERSION", raising=False)
     config_dict = {
         "applications": [
             {

tests/unit_tests/io/test_io_handler.py

@@ -86,6 +86,20 @@ def test_resolve_test_resource_path_macros_nested_structures(tmp_test_directory)
     assert resolved["plain"] == "no_macro"
 
 
+def test_resolve_test_resource_paths_uses_versioned_environment_resources(
+    tmp_test_directory, monkeypatch
+):
+    tests_path = tmp_test_directory / "simtools-tests"
+    monkeypatch.delenv("SIMTOOLS_TEST_RESOURCES", raising=False)
+    monkeypatch.setenv("SIMTOOLS_TESTS_PATH", str(tests_path))
+    monkeypatch.setenv("SIMTOOLS_TESTS_TAG", "v0.37.0")
+
+    resolved = io_handler_module.resolve_test_resource_paths("${downloaded:corsika_limits.ecsv}")
+
+    expected_root = tests_path / "v0.37.0" / "integration_tests"
+    assert resolved == str(expected_root / "downloaded/corsika_limits.ecsv")
+
+
 def test_get_model_configuration_directory(args_dict, io_handler):
     model_version = "1.0.0"
     label = "test-io-handler"

tests/unit_tests/production_configuration/test_job_grid_io.py

@@ -86,6 +86,39 @@ def test_serialize_and_read_job_grid_with_optional_string_fields(tmp_test_direct
     assert read_rows[1]["scan_label"] == "asum220"
 
 
+def test_serialize_job_grid_sorts_rows_by_production_configuration(tmp_test_directory):
+    output_file = Path(tmp_test_directory) / "job_grid.ecsv"
+    rows = [
+        {
+            **_job_rows()[0],
+            "run_number": 12,
+            "primary": "proton",
+            "azimuth_angle": 10 * u.deg,
+        },
+        {
+            **_job_rows()[0],
+            "run_number": 11,
+            "primary": "gamma",
+            "azimuth_angle": 20 * u.deg,
+        },
+        {
+            **_job_rows()[0],
+            "run_number": 10,
+            "primary": "gamma",
+            "azimuth_angle": 10 * u.deg,
+        },
+    ]
+
+    job_grid_io.serialize_job_grid(rows, output_file, metadata=_metadata())
+    output_table = Table.read(output_file, format="ascii.ecsv")
+
+    assert list(zip(output_table["primary"], output_table["azimuth_angle"], strict=True)) == [
+        ("gamma", 10.0),
+        ("gamma", 20.0),
+        ("proton", 10.0),
+    ]
+
+
 def test_serialize_job_grid_rejects_non_ecsv_output(tmp_test_directory):
     output_file = Path(tmp_test_directory) / "job_grid.txt"