From ae1e6934899832a5300215cb2daf91406bac9a4c Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 13 May 2026 15:00:03 -0400 Subject: [PATCH 1/8] Add `common_paths` to input dataset config for non-subject files Replaces the hard-coded `dataset_description.json` in sparse-checkout, `datalad get`, and `datalad run -i` with a configurable `common_paths` list on each input dataset entry. Defaults to `["dataset_description.json"]` to preserve existing behaviour. An empty list disables all common-path inclusion. Closes #374 Co-Authored-By: Claude Sonnet 4.6 --- babs/input_dataset.py | 9 +++++++++ babs/templates/participant_job.sh.jinja2 | 9 ++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/babs/input_dataset.py b/babs/input_dataset.py index 45ec2ed6..ec138032 100644 --- a/babs/input_dataset.py +++ b/babs/input_dataset.py @@ -24,6 +24,7 @@ def __init__( is_zipped, unzipped_path_containing_subject_dirs=None, required_files=None, + common_paths=None, processing_level=None, babs_project_analysis_path=None, ): @@ -43,6 +44,11 @@ def __init__( when unzipped, this string precedes the subject directories required_files: list of str or None list of required files in the input dataset + common_paths: list of str or None + paths relative to the dataset root to include in the sparse-checkout for every job, + in addition to the per-subject (and per-session) path. + Defaults to ``["dataset_description.json"]`` when ``None``. + Pass an empty list to disable all common-path inclusion. processing_level: {'subject', 'session'} or None whether processing is done on a subject-wise or session-wise basis babs_project_analysis_path: str or None @@ -57,6 +63,7 @@ def __init__( else: self.is_zipped = bool(is_zipped) self.required_files = required_files + self.common_paths = ['dataset_description.json'] if common_paths is None else common_paths if processing_level not in ['subject', 'session']: raise ValueError('invalid `processing_level`!') self.processing_level = processing_level @@ -269,6 +276,7 @@ def as_dict(self): 'is_zipped': self.is_zipped, 'unzipped_path_containing_subject_dirs': unzipped_path, 'required_files': self.required_files, + 'common_paths': self.common_paths, 'processing_level': self.processing_level, 'babs_project_analysis_path': self.babs_project_analysis_path, } @@ -433,4 +441,5 @@ def __init__(self, input_dataset): input_dataset.unzipped_path_containing_subject_dirs ) self.required_files = input_dataset.required_files + self.common_paths = input_dataset.common_paths self.processing_level = input_dataset.processing_level diff --git a/babs/templates/participant_job.sh.jinja2 b/babs/templates/participant_job.sh.jinja2 index f3151cea..5a3683fa 100644 --- a/babs/templates/participant_job.sh.jinja2 +++ b/babs/templates/participant_job.sh.jinja2 @@ -87,7 +87,9 @@ echo "# Pull down the input session but don't retrieve data contents:" {% if not input_dataset['is_zipped'] %} datalad get -n "{{ input_dataset['path_in_babs'] }}/{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}" -datalad get -n "{{ input_dataset['path_in_babs'] }}/dataset_description.json" +{% for common_path in input_dataset['common_paths'] %} +datalad get -n "{{ input_dataset['path_in_babs'] }}/{{ common_path }}" +{% endfor %} {% else %} datalad get -n "{{ input_dataset['path_in_babs'] }}" {% endif %} @@ -99,7 +101,7 @@ datalad get -n "{{ input_dataset['path_in_babs'] }}" if [ -d "{{ input_dataset['path_in_babs'] }}/.git" ]; then ( cd "{{ input_dataset['path_in_babs'] }}" && \ ( git sparse-checkout init --no-cone 2>/dev/null && \ - { echo "{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}"; echo 'dataset_description.json'; } | git sparse-checkout set --stdin 2>/dev/null ) ) || true + { echo "{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}"; {% for common_path in input_dataset['common_paths'] %}echo '{{ common_path }}'; {% endfor %}} | git sparse-checkout set --stdin 2>/dev/null ) ) || true fi {% endif %} {% endfor %} @@ -137,7 +139,8 @@ datalad run \ {% for input_dataset in input_datasets %} {% if not input_dataset['is_zipped'] %} -i "{{ input_dataset['unzipped_path_containing_subject_dirs'] }}/{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}" \ - -i "{{ input_dataset['path_in_babs'] }}/dataset_description.json" \ +{% for common_path in input_dataset['common_paths'] %} -i "{{ input_dataset['path_in_babs'] }}/{{ common_path }}" \ +{% endfor %} {% else %} -i "${%raw%}{{%endraw%}{{ input_dataset['name'].upper() }}_ZIP{%raw%}}{%endraw%}" \ {% endif %} From fb21ddfa7f17a90925b691f9013a92d924efde2d Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 13 May 2026 16:00:21 -0400 Subject: [PATCH 2/8] adding echo --- babs/templates/participant_job.sh.jinja2 | 1 + 1 file changed, 1 insertion(+) diff --git a/babs/templates/participant_job.sh.jinja2 b/babs/templates/participant_job.sh.jinja2 index 5a3683fa..5f26e886 100644 --- a/babs/templates/participant_job.sh.jinja2 +++ b/babs/templates/participant_job.sh.jinja2 @@ -88,6 +88,7 @@ echo "# Pull down the input session but don't retrieve data contents:" datalad get -n "{{ input_dataset['path_in_babs'] }}/{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}" {% for common_path in input_dataset['common_paths'] %} +echo "# Getting common path: {{ input_dataset['path_in_babs'] }}/{{ common_path }}" datalad get -n "{{ input_dataset['path_in_babs'] }}/{{ common_path }}" {% endfor %} {% else %} From e3dafc819cbb73e337139e6955dd717d0292c1d4 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Wed, 13 May 2026 16:18:41 -0400 Subject: [PATCH 3/8] docs: document common_paths field in preparation_config_yaml_file.rst Add common_paths to the section overview list and optional sections list, add a stub required_files section, and add a full common_paths section with examples and usage notes. Co-Authored-By: Claude Sonnet 4.6 --- docs/preparation_config_yaml_file.rst | 64 ++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/preparation_config_yaml_file.rst b/docs/preparation_config_yaml_file.rst index 767a95ac..e2593aea 100644 --- a/docs/preparation_config_yaml_file.rst +++ b/docs/preparation_config_yaml_file.rst @@ -29,6 +29,7 @@ Sections in the configuration YAML file * **all_results_in_one_zip**: whether to zip all results in one zip file; * **zip_foldernames**: the results foldername(s) to be zipped; * **required_files**: to only keep subjects (sessions) that have this list of required files in input dataset(s); +* **common_paths**: dataset-root paths to include in the sparse-checkout for every job, in addition to the per-subject path (e.g., a shared ``phenotype/participants.tsv`` file); * **alert_log_messages**: alert messages in the log files that may be helpful for debugging errors in failed jobs; Among these sections, these sections are optional: @@ -40,6 +41,7 @@ Among these sections, these sections are optional: * You must include this section if there are more one input dataset. * **required_files** +* **common_paths** * **alert_log_messages** * **imported_files** @@ -103,7 +105,7 @@ Example section **input_datasets** unzipped_path_containing_subject_dirs: "freesurfer" path_in_babs: inputs/data/freesurfer -This example shows two input datasets: +This example shows two input datasets: one is a raw BIDS dataset, and the other is a zipped FreeSurfer results from another BABS project. Previously, the commandline to use something like this would have required:: @@ -773,4 +775,64 @@ Notes: .. _required_files: +Section ``required_files`` +========================== + +.. note:: + + ``required_files`` is currently not fully implemented. + The field is accepted in the YAML file but filtering is not yet applied. + +.. _common-paths: + +Section ``common_paths`` +========================= + +The ``common_paths`` field lists paths (relative to an input dataset's root) +that every job should include in the sparse-checkout and retrieve with +``datalad get``, in addition to the per-subject (and per-session) path. +This is useful when BIDS Apps or processing scripts need dataset-level files +that live outside any individual subject directory. + +By default (when the field is omitted), BABS automatically includes +``dataset_description.json`` for every non-zipped input dataset. +Once you supply ``common_paths`` explicitly, the default is **replaced** — +so if you still want ``dataset_description.json`` you must list it yourself. + +``common_paths`` is optional. It is nested under the relevant input dataset +entry inside the ``input_datasets`` section. + +Example — keep the default ``dataset_description.json`` **and** add a shared +phenotype file: + +.. code-block:: yaml + + input_datasets: + BIDS: + is_zipped: false + origin_url: "/path/to/BIDS" + path_in_babs: inputs/data/BIDS + common_paths: + - "phenotype/participants.tsv" + - "dataset_description.json" + +Example — disable all common-path retrieval (pass an empty list): + +.. code-block:: yaml + + input_datasets: + BIDS: + is_zipped: false + origin_url: "/path/to/BIDS" + path_in_babs: inputs/data/BIDS + common_paths: [] + +Notes: + +* Paths are relative to the input dataset root (e.g., ``"phenotype/participants.tsv"`` + not ``"inputs/data/BIDS/phenotype/participants.tsv"``). +* Each path is retrieved individually with ``datalad get -n`` so you can track + exactly which files are fetched in the job log. +* This field has no effect on zipped input datasets. + From 443dcf66a0d2a0b79ee8a99fddceced9186639ac Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Thu, 21 May 2026 08:59:56 -0500 Subject: [PATCH 4/8] Fixup tests Add common paths to the submit script test inputs. No need to add to the zipped ones, they are skipped. --- tests/test_generate_submit_script.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_generate_submit_script.py b/tests/test_generate_submit_script.py index 8f4e832b..7ee9f610 100644 --- a/tests/test_generate_submit_script.py +++ b/tests/test_generate_submit_script.py @@ -14,6 +14,7 @@ 'path_in_babs': 'inputs/data/BIDS', 'unzipped_path_containing_subject_dirs': 'inputs/data/BIDS', 'is_zipped': False, + 'common_paths': ['dataset_description.json'], }, ] @@ -29,6 +30,7 @@ 'path_in_babs': 'inputs/data/BIDS', 'unzipped_path_containing_subject_dirs': 'inputs/data/BIDS', 'is_zipped': False, + 'common_paths': [], }, ] From da538a9082be37414dcaa8b93d7c373070988772 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Tue, 23 Jun 2026 13:02:33 -0500 Subject: [PATCH 5/8] participant_job: grab BIDS-inherited metadata for every job `common_paths` (PR #376) replaces the hard-coded `dataset_description.json` but still drops root-level inherited BIDS sidecars (e.g. a top-level `task-*_bold.json` carrying RepetitionTime/PhaseEncodingDirection) from the per-job sparse-checkout, so fmriprep fails on valid data with `KeyError: 'RepetitionTime'` / "Missing readout timing information". Make BIDS inheritance automatic and positional: each job now grabs every metadata blob at the tiers ABOVE its checkout -- the dataset root always, plus the subject tier (sub-XX/) for session-level jobs, whose sub-XX/ses-YY checkout would otherwise miss files sitting directly under sub-XX/. In valid BIDS no data files sit at those tiers, so every blob there is inheritable metadata. resolve_tier() uses `git ls-tree HEAD` -- non-recursive (anchored, never crosses into other subjects) and reading the committed tree (independent of sparse/checkout state) -- and resolves to literal paths because datalad/sparse-checkout cannot take a glob. The resolved paths feed all three consumers: `datalad get -n`, `git sparse-checkout set --stdin`, and `datalad run -i`. `common_paths` is kept as the explicit-extras escape hatch for a non-inherited file the grab doesn't reach (e.g. `sourcedata/.../nidm.ttl`); its default flips from `['dataset_description.json']` to `[]` since inheritance now covers `dataset_description.json` automatically. Co-Authored-By: Claude Opus 4.8 (1M context) --- babs/input_dataset.py | 12 +++--- babs/templates/participant_job.sh.jinja2 | 47 +++++++++++++++++++----- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/babs/input_dataset.py b/babs/input_dataset.py index ec138032..45aba31d 100644 --- a/babs/input_dataset.py +++ b/babs/input_dataset.py @@ -45,10 +45,12 @@ def __init__( required_files: list of str or None list of required files in the input dataset common_paths: list of str or None - paths relative to the dataset root to include in the sparse-checkout for every job, - in addition to the per-subject (and per-session) path. - Defaults to ``["dataset_description.json"]`` when ``None``. - Pass an empty list to disable all common-path inclusion. + *explicit* dataset-root-relative paths to include in the sparse-checkout for + every job, in addition to the per-subject (and per-session) path AND the + automatic BIDS-inheritance grab (see ``participant_job.sh.jinja2``: every job + also pulls all metadata blobs sitting at the dataset root, plus the subject + tier for session-level jobs). Use this only for a non-inherited file the grab + doesn't reach (e.g. a shared ``sourcedata/.../nidm.ttl``). Defaults to ``[]``. processing_level: {'subject', 'session'} or None whether processing is done on a subject-wise or session-wise basis babs_project_analysis_path: str or None @@ -63,7 +65,7 @@ def __init__( else: self.is_zipped = bool(is_zipped) self.required_files = required_files - self.common_paths = ['dataset_description.json'] if common_paths is None else common_paths + self.common_paths = [] if common_paths is None else common_paths if processing_level not in ['subject', 'session']: raise ValueError('invalid `processing_level`!') self.processing_level = processing_level diff --git a/babs/templates/participant_job.sh.jinja2 b/babs/templates/participant_job.sh.jinja2 index 5f26e886..32e06916 100644 --- a/babs/templates/participant_job.sh.jinja2 +++ b/babs/templates/participant_job.sh.jinja2 @@ -83,27 +83,53 @@ git checkout -f # pull down only needed session path and explicit dataset-level metadata: echo "# Pull down the input session but don't retrieve data contents:" + +# resolve_tier lists the files of ONE directory tier of an input subdataset. +# git ls-tree reads the committed tree (independent of sparse/checkout state) and is +# non-recursive, so it is anchored to the named tier and never descends into other +# subjects. In valid BIDS no data files sit at the dataset root or directly under +# sub-XX/, so every blob at those tiers is BIDS-inherited metadata (task/modality +# .json sidecars, participants/sessions .tsv, .bval/.bvec, ...). We resolve to literal +# paths because `datalad get`/`git sparse-checkout` cannot take a glob. +resolve_tier() { # $1 = subdataset path, $2 = tier dir relative to its root ('' = root) + git -C "$1" ls-tree HEAD ${2:+"$2/"} 2>/dev/null | awk '$2 == "blob" { sub(/^[^\t]*\t/, ""); print }' +} + +DATALAD_INPUTS=() {% for input_dataset in input_datasets %} {% if not input_dataset['is_zipped'] %} -datalad get -n "{{ input_dataset['path_in_babs'] }}/{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}" +datalad get -n "{{ input_dataset['path_in_babs'] }}/${subid}{% if processing_level == 'session' %}/${sesid}{% endif %}" +# BIDS inheritance: pull metadata from the tiers ABOVE this job's checkout -- the +# dataset root always, plus the subject tier (sub-XX/) for session-level jobs, whose +# sub-XX/ses-YY checkout would otherwise miss files sitting directly under sub-XX/. +inherited=() +while IFS= read -r _f; do inherited+=( "$_f" ); done < <(resolve_tier "{{ input_dataset['path_in_babs'] }}" "") +{% if processing_level == 'session' %} +while IFS= read -r _f; do inherited+=( "$_f" ); done < <(resolve_tier "{{ input_dataset['path_in_babs'] }}" "${subid}") +{% endif %} +for rel in ${inherited[@]+"${inherited[@]}"}; do + echo "# Getting inherited metadata: {{ input_dataset['path_in_babs'] }}/${rel}" + datalad get -n "{{ input_dataset['path_in_babs'] }}/${rel}" + DATALAD_INPUTS+=( -i "{{ input_dataset['path_in_babs'] }}/${rel}" ) +done {% for common_path in input_dataset['common_paths'] %} echo "# Getting common path: {{ input_dataset['path_in_babs'] }}/{{ common_path }}" datalad get -n "{{ input_dataset['path_in_babs'] }}/{{ common_path }}" {% endfor %} -{% else %} -datalad get -n "{{ input_dataset['path_in_babs'] }}" -{% endif %} -{% endfor %} -# Restrict each BIDS input subdataset to the current subject so BIDS apps that index the full dataset (e.g. pybids BIDSLayout) don't try to read other subjects' files, which may not be retrieved -{% for input_dataset in input_datasets %} -{% if not input_dataset['is_zipped'] %} +# Restrict this subdataset to the subject/session subtree + inherited metadata + any +# explicit common_paths, so BIDS apps that index the dataset (e.g. pybids BIDSLayout) +# don't read other subjects' files, which may not be retrieved. if [ -d "{{ input_dataset['path_in_babs'] }}/.git" ]; then + sparse=( "${subid}{% if processing_level == 'session' %}/${sesid}{% endif %}" {% for common_path in input_dataset['common_paths'] %}'{{ common_path }}' {% endfor %}) + sparse+=( ${inherited[@]+"${inherited[@]}"} ) ( cd "{{ input_dataset['path_in_babs'] }}" && \ - ( git sparse-checkout init --no-cone 2>/dev/null && \ - { echo "{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}"; {% for common_path in input_dataset['common_paths'] %}echo '{{ common_path }}'; {% endfor %}} | git sparse-checkout set --stdin 2>/dev/null ) ) || true + git sparse-checkout init --no-cone 2>/dev/null && \ + printf '%s\n' "${sparse[@]}" | git sparse-checkout set --stdin 2>/dev/null ) || true fi +{% else %} +datalad get -n "{{ input_dataset['path_in_babs'] }}" {% endif %} {% endfor %} @@ -146,6 +172,7 @@ datalad run \ -i "${%raw%}{{%endraw%}{{ input_dataset['name'].upper() }}_ZIP{%raw%}}{%endraw%}" \ {% endif %} {% endfor %} + ${DATALAD_INPUTS[@]+"${DATALAD_INPUTS[@]}"} \ {% for image_path in container_image_paths %} -i "{{ image_path }}" \ {% endfor %} From 4945e6d4e4df6ca3f38ccabddc8f6edfab7b4e02 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Tue, 23 Jun 2026 13:12:51 -0500 Subject: [PATCH 6/8] tests: cover BIDS-inheritance grab + common_paths default Render-string assertions (matching this module's existing render+shellcheck style): root tier resolved for every job, subject tier only for session-level jobs, zipped inputs skipped, and an explicit common_paths entry reaching `datalad get` / sparse-checkout / `datalad run -i`. Plus an object-level test that common_paths defaults to []. These check the generated script text; runtime resolve_tier behavior is left to the e2e walkthrough (a faithful staging assertion is awkward until the splice-point hooks land -- see PR description / follow-up). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_generate_submit_script.py | 58 ++++++++++++++++++++++++++++ tests/test_input_datasets.py | 24 ++++++++++++ 2 files changed, 82 insertions(+) diff --git a/tests/test_generate_submit_script.py b/tests/test_generate_submit_script.py index 7ee9f610..d57121ee 100644 --- a/tests/test_generate_submit_script.py +++ b/tests/test_generate_submit_script.py @@ -176,3 +176,61 @@ def test_generate_submit_script_pipeline(tmp_path): if not passed: print(script_content) assert passed, status + + +def _render(input_datasets, processing_level): + """Render the participant job for a minimal single-app config (no YAML needed).""" + return generate_submit_script( + queue_system='slurm', + cluster_resources_config={'interpreting_shell': '/bin/bash'}, + script_preamble='', + job_scratch_directory='/tmp/job', + input_datasets=input_datasets, + processing_level=processing_level, + container_name='fmriprep', + zip_foldernames={'fmriprep': '0'}, + container_images=['containers/fmriprep.sif'], + ) + + +def test_bids_inheritance_tiers_per_processing_level(): + """Root tier is grabbed for every job; the subject tier only for session jobs. + + These assert on the *generated script text* (like the rest of this module); + the runtime ``resolve_tier`` behavior is exercised by the e2e walkthrough. + """ + path = input_datasets_prep[0]['path_in_babs'] + subj = _render(input_datasets_prep, 'subject') + sess = _render(input_datasets_prep, 'session') + + # root tier resolved for both levels + assert f'resolve_tier "{path}" ""' in subj + assert f'resolve_tier "{path}" ""' in sess + + # subject tier resolved ONLY for session-level jobs (the gap session checkout misses) + assert f'resolve_tier "{path}" "${{subid}}"' in sess + assert f'resolve_tier "{path}" "${{subid}}"' not in subj + + # resolved paths are wired into `datalad run` as inputs + for script in (subj, sess): + assert 'DATALAD_INPUTS=()' in script + assert '${DATALAD_INPUTS[@]+"${DATALAD_INPUTS[@]}"}' in script + + +def test_bids_inheritance_skips_zipped_inputs(): + """Zipped inputs get no inheritance grab; only the unzipped one is resolved.""" + script = _render(input_datasets_fmriprep_ingressed_anat, 'subject') + # the unzipped BIDS dataset is resolved ... + assert 'resolve_tier "inputs/data/BIDS" ""' in script + # ... but the zipped freesurfer dataset (path_in_babs 'inputs/data') is not + assert 'resolve_tier "inputs/data" ""' not in script + + +def test_common_paths_threaded_into_consumers(): + """An explicit common_paths entry reaches get, sparse-checkout, and `datalad run -i`.""" + dss = [dict(input_datasets_prep[0], common_paths=['phenotype/participants.tsv'])] + script = _render(dss, 'subject') + full = 'inputs/data/BIDS/phenotype/participants.tsv' + assert f'datalad get -n "{full}"' in script # datalad get + assert "'phenotype/participants.tsv'" in script # sparse=( ... ) set + assert f'-i "{full}"' in script # datalad run input diff --git a/tests/test_input_datasets.py b/tests/test_input_datasets.py index dfa9c47a..e901a186 100644 --- a/tests/test_input_datasets.py +++ b/tests/test_input_datasets.py @@ -5,6 +5,30 @@ from babs.input_datasets import InputDatasets +def _bids(**overrides): + kwargs = { + 'name': 'BIDS', + 'origin_url': '/does/not/matter', + 'path_in_babs': 'inputs/data/BIDS', + 'is_zipped': False, + 'processing_level': 'subject', + } + kwargs.update(overrides) + return InputDataset(**kwargs) + + +def test_common_paths_default_is_empty(): + """common_paths defaults to [] (BIDS inheritance is automatic, not via this field).""" + # default: no common_paths supplied + assert _bids().common_paths == [] + # an explicit list is preserved as-is ... + assert _bids(common_paths=['phenotype/participants.tsv']).common_paths == [ + 'phenotype/participants.tsv' + ] + # ... including an explicit empty list + assert _bids(common_paths=[]).common_paths == [] + + @pytest.mark.parametrize( ('session_type', 'processing_level'), [ From 821a057aa21bb968a22a51154679129e1e0e63fb Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Tue, 23 Jun 2026 14:18:39 -0500 Subject: [PATCH 7/8] docs: document automatic BIDS inheritance; reframe common_paths The fixup makes BIDS metadata inheritance automatic, so dataset_description.json and other upper-tier sidecars are staged without configuration, and common_paths now defaults to []. Add a "BIDS metadata inheritance (automatic)" section and recast common_paths as the explicit-extras escape hatch (e.g. a non-inherited sourcedata/.../nidm.ttl), replacing the old dataset_description.json-default and "keep the default + add" examples. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/preparation_config_yaml_file.rst | 72 ++++++++++++++++----------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/docs/preparation_config_yaml_file.rst b/docs/preparation_config_yaml_file.rst index e2593aea..da5edd03 100644 --- a/docs/preparation_config_yaml_file.rst +++ b/docs/preparation_config_yaml_file.rst @@ -29,7 +29,7 @@ Sections in the configuration YAML file * **all_results_in_one_zip**: whether to zip all results in one zip file; * **zip_foldernames**: the results foldername(s) to be zipped; * **required_files**: to only keep subjects (sessions) that have this list of required files in input dataset(s); -* **common_paths**: dataset-root paths to include in the sparse-checkout for every job, in addition to the per-subject path (e.g., a shared ``phenotype/participants.tsv`` file); +* **common_paths**: extra, *non-inherited* files to stage for every job (BIDS metadata inheritance is automatic — see :ref:`bids-inheritance`); e.g. a shared ``sourcedata/NIDM/nidm.ttl``; * **alert_log_messages**: alert messages in the log files that may be helpful for debugging errors in failed jobs; Among these sections, these sections are optional: @@ -783,40 +783,49 @@ Section ``required_files`` ``required_files`` is currently not fully implemented. The field is accepted in the YAML file but filtering is not yet applied. -.. _common-paths: +.. _bids-inheritance: -Section ``common_paths`` -========================= +BIDS metadata inheritance (automatic) +===================================== -The ``common_paths`` field lists paths (relative to an input dataset's root) -that every job should include in the sparse-checkout and retrieve with -``datalad get``, in addition to the per-subject (and per-session) path. -This is useful when BIDS Apps or processing scripts need dataset-level files -that live outside any individual subject directory. +For every non-zipped input dataset, BABS automatically stages the BIDS +*inherited* metadata each job needs, in addition to the per-subject (and +per-session) path. Under the `BIDS Inheritance Principle +`_, +a metadata file that sits *above* a data file in the directory hierarchy applies +to it. A per-subject (or per-session) sparse-checkout would otherwise drop those +upper-tier files — e.g. a top-level ``task-rest_bold.json`` carrying +``RepetitionTime``/``PhaseEncodingDirection``, ``participants.tsv``, or +``dataset_description.json`` — making BIDS Apps fail on otherwise-valid data. -By default (when the field is omitted), BABS automatically includes -``dataset_description.json`` for every non-zipped input dataset. -Once you supply ``common_paths`` explicitly, the default is **replaced** — -so if you still want ``dataset_description.json`` you must list it yourself. +For each job, BABS includes every metadata file at the tiers *above* the job's +checkout: -``common_paths`` is optional. It is nested under the relevant input dataset -entry inside the ``input_datasets`` section. +* the **dataset root** — always; +* the **subject tier** (``sub-XX/``) — additionally for ``session``-level jobs, + whose ``sub-XX/ses-YY`` checkout would otherwise miss files sitting directly + under ``sub-XX/`` (subject-level jobs already check out the whole ``sub-XX/`` + subtree). -Example — keep the default ``dataset_description.json`` **and** add a shared -phenotype file: +This is automatic and needs no configuration: ``dataset_description.json`` and +other inherited sidecars are staged out of the box. -.. code-block:: yaml +.. _common-paths: - input_datasets: - BIDS: - is_zipped: false - origin_url: "/path/to/BIDS" - path_in_babs: inputs/data/BIDS - common_paths: - - "phenotype/participants.tsv" - - "dataset_description.json" +Section ``common_paths`` +========================= + +``common_paths`` is an optional escape hatch for **extra, non-inherited** files +that the automatic inheritance grab does not reach — for example a specific file +in a subdirectory, such as a shared ``sourcedata/NIDM/nidm.ttl``. It lists paths +relative to an input dataset's root; each is added to the job's sparse-checkout, +retrieved with ``datalad get -n``, and recorded as a ``datalad run`` input. -Example — disable all common-path retrieval (pass an empty list): +It defaults to ``[]`` — BIDS inheritance already covers the dataset-level +sidecars, so valid BIDS needs nothing extra here. It is nested under the +relevant input dataset entry inside the ``input_datasets`` section. + +Example — also stage a specific shared file the inheritance grab won't reach: .. code-block:: yaml @@ -825,14 +834,17 @@ Example — disable all common-path retrieval (pass an empty list): is_zipped: false origin_url: "/path/to/BIDS" path_in_babs: inputs/data/BIDS - common_paths: [] + common_paths: + - "sourcedata/NIDM/nidm.ttl" Notes: -* Paths are relative to the input dataset root (e.g., ``"phenotype/participants.tsv"`` - not ``"inputs/data/BIDS/phenotype/participants.tsv"``). +* Paths are relative to the input dataset root (e.g., ``"sourcedata/NIDM/nidm.ttl"`` + not ``"inputs/data/BIDS/sourcedata/NIDM/nidm.ttl"``). * Each path is retrieved individually with ``datalad get -n`` so you can track exactly which files are fetched in the job log. * This field has no effect on zipped input datasets. +* BIDS metadata inheritance is automatic (see above) — you do **not** need to + list ``dataset_description.json`` or other inherited sidecars here. From 3d112156a13a189f3c71c4e3883f0397f9be3544 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Tue, 21 Jul 2026 18:25:02 -0500 Subject: [PATCH 8/8] test fixup for 369 --- tests/test_generate_submit_script.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_generate_submit_script.py b/tests/test_generate_submit_script.py index 7dcf1bb7..95929ba8 100644 --- a/tests/test_generate_submit_script.py +++ b/tests/test_generate_submit_script.py @@ -192,6 +192,7 @@ def _render(input_datasets, processing_level): container_name='fmriprep', zip_foldernames={'fmriprep': '0'}, container_images=['containers/fmriprep.sif'], + analysis_path='/tmp/babs_project/analysis', )