Preflight memory

brainscore_vision/__init__.py

@@ -1,11 +1,13 @@
 import logging
+import os
 from typing import Dict, Any, Union, Callable
 
 from brainscore_core.supported_data_standards.brainio.assemblies import DataAssembly
 from brainscore_core.supported_data_standards.brainio.stimuli import StimulusSet
 
 from brainscore_core.benchmarks import Benchmark
 from brainscore_core.metrics import Metric, Score
+from brainscore_core.benchmarks import score_benchmark
 from brainscore_core.plugin_management.conda_score import wrap_score
 from brainscore_core.plugin_management.import_plugin import import_plugin
 from brainscore_vision.metrics import Ceiling
@@ -74,7 +76,21 @@ def _run_score(model_identifier: str, benchmark_identifier: str) -> Score:
     """
     model: BrainModel = load_model(model_identifier)
     benchmark: Benchmark = load_benchmark(benchmark_identifier)
-    score: Score = benchmark(model)
+    try:
+        score: Score = score_benchmark(benchmark, model)
+    except AssertionError:
+        cache_dir = os.path.expanduser(
+            '~/.result_caching/brainscore_vision.model_helpers.activations.core'
+            '.ActivationsExtractorHelper._from_paths_stored'
+        )
+        raise AssertionError(
+            f"Activations cache has stale paths — the cached stimulus paths no longer match "
+            f"the current stimulus locations (e.g. temp directory changed between runs).\n\n"
+            f"Fix: delete the stale cache entry and re-run:\n"
+            f"  rm {cache_dir}/identifier={model_identifier},stimuli_identifier=*.pkl\n\n"
+            f"Or to clear the entire activations cache:\n"
+            f"  rm {cache_dir}/*.pkl"
+        )
     score.attrs['model_identifier'] = model_identifier
     score.attrs['benchmark_identifier'] = benchmark_identifier
     try:  # attempt to look up the layer commitment if model uses a standard layer model
@@ -104,3 +120,7 @@ def score(model_identifier: str, benchmark_identifier: str, conda_active: bool =
     return wrap_score(__file__,
                       model_identifier=model_identifier, benchmark_identifier=benchmark_identifier,
                       score_function=_run_score, conda_active=conda_active)
+
+
+# Public re-export so callers can do: from brainscore_vision import preallocate_memory
+from brainscore_vision.benchmark_helpers.memory import preallocate_memory  # noqa: E402

brainscore_vision/benchmark_helpers/benchmark_costs.json

@@ -0,0 +1,51 @@
+{
+  "Allen2022_fmri.IT-ridge": 0.4113,
+  "Allen2022_fmri.V1-ridge": 0.5544,
+  "Allen2022_fmri.V2-ridge": 0.7258,
+  "Allen2022_fmri.V4-ridge": 0.3258,
+  "Allen2022_fmri_4subj.IT-ridge": 1.49,
+  "Allen2022_fmri_4subj.V1-ridge": 0.6966,
+  "Allen2022_fmri_4subj.V2-ridge": 1.3744,
+  "Allen2022_fmri_4subj.V4-ridge": 0.6729,
+  "Allen2022_fmri_surface.IT-ridge": 0.6236,
+  "Allen2022_fmri_surface.V1-ridge": 1.2537,
+  "Allen2022_fmri_surface.V2-ridge": 1.5052,
+  "Allen2022_fmri_surface.V4-ridge": 0.6126,
+  "Allen2022_fmri_surface_4subj.IT-ridge": 0.5683,
+  "Allen2022_fmri_surface_4subj.V1-ridge": 1.3479,
+  "Allen2022_fmri_surface_4subj.V2-ridge": 1.6253,
+  "Allen2022_fmri_surface_4subj.V4-ridge": 0.7984,
+  "Cadena2017-mask": 4.4728,
+  "Cadena2017-pls": 6.9435,
+  "FreemanZiemba2013.V1-pls": 4.0874,
+  "FreemanZiemba2013.V2-pls": 3.8205,
+  "FreemanZiemba2013public.V1-pls": 1.813,
+  "FreemanZiemba2013public.V2-pls": 0.8573,
+  "Gifford2022.IT-ridge": 3.6693,
+  "Gifford2022.IT-ridgecv": 7.3324,
+  "Hebart2023_fmri.IT-ridge": 16.8948,
+  "Hebart2023_fmri.IT-ridgecv": 3.8821,
+  "Hebart2023_fmri.V1-ridge": 4.843,
+  "Hebart2023_fmri.V1-ridgecv": 8.8359,
+  "Hebart2023_fmri.V2-ridge": 15.5814,
+  "Hebart2023_fmri.V2-ridgecv": 12.4336,
+  "Hebart2023_fmri.V4-ridge": 6.8122,
+  "Hebart2023_fmri.V4-ridgecv": 12.4122,
+  "Igustibagus2024-ridge": 2.1582,
+  "MajajHong2015.IT-pls": 2.8336,
+  "MajajHong2015.V4-pls": 4.0503,
+  "MajajHong2015public.IT-pls": 3.5527,
+  "MajajHong2015public.V4-pls": 4.9696,
+  "Papale2025.IT-ridge": 43.45,
+  "Papale2025.IT-ridgecv": 7.22,
+  "Papale2025.V1-ridge": 15.07,
+  "Papale2025.V1-ridgecv": 26.58,
+  "Papale2025.V4-ridge": 40.54,
+  "Rajalingham2020.IT-pls": 0.45,
+  "Sanghavi2020.IT-pls": 6.71,
+  "Sanghavi2020.V4-pls": 8.97,
+  "SanghaviJozwik2020.IT-pls": 5.41,
+  "SanghaviJozwik2020.V4-pls": 7.65,
+  "SanghaviMurty2020.IT-pls": 0.22,
+  "SanghaviMurty2020.V4-pls": 0.56
+}