From fb622a97045a3f3e4c9b1cf0a2910bcc06f800ba Mon Sep 17 00:00:00 2001 From: Timo Diepers Date: Fri, 21 Aug 2026 08:18:26 +0200 Subject: [PATCH] Store scenario metadata on exported Brightway databases (#1) * Store scenario metadata on exported Brightway databases Databases written to Brightway only encoded the IAM model, pathway and year in their name. Attach them to the Brightway database metadata instead, together with an ISO 8601 `representative_time` timestamp of the point in time the database represents, the ecoinvent version, the system model and the premise version. Superstructure and scenario-array databases list their scenarios under `scenarios`, and only carry `year`/`representative_time` at the top level when all their scenarios share the same year. * Address review: drop redundant year, expand docs example - `year` is dropped from the metadata: it is already carried by the ISO 8601 `representative_time` timestamp. - The changelog entry moves under `[Unreleased]`. - The docs example shows the fields brightway writes itself next to the ones premise adds, and notes that `bd.databases["db_name"]` and `bd.Database("db_name").metadata` are the same mapping. - The `metadata` argument docstrings are removed from both writers. --- CHANGELOG.md | 9 ++ docs/load.rst | 48 +++++++++ premise/brightway2.py | 10 ++ premise/brightway25.py | 11 +++ premise/new_database.py | 22 +++++ premise/utils.py | 98 +++++++++++++++++++ tests/test_database_metadata.py | 166 ++++++++++++++++++++++++++++++++ tests/test_new_database.py | 51 ++++++---- 8 files changed, 395 insertions(+), 20 deletions(-) create mode 100644 tests/test_database_metadata.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b6d9ce..70591298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project are documented in this file. +## [Unreleased] + +### Added +- Databases exported to Brightway now carry scenario metadata in + `bw2data.databases[name]`: `iam_model`, `pathway`, `representative_time` + (ISO 8601), `ecoinvent_version`, `system_model`, `premise_version` and, + if any, `external_scenarios`. Superstructure and scenario-array databases + list their scenarios under `scenarios`. + ## [2.4.9.2] ### Added diff --git a/docs/load.rst b/docs/load.rst index ae5d9912..cec296bd 100644 --- a/docs/load.rst +++ b/docs/load.rst @@ -27,6 +27,54 @@ If several databases have been built, the user can give them specific names, lik biosphere database already registered. +Scenario metadata +***************** + +Every database written to brightway carries, in its brightway metadata, a description +of the scenario and of the point in time it represents. It sits next to the fields +brightway maintains itself: + +.. code-block:: python + + import bw2data as bd + + bd.databases["ei_cutoff_3.10.1_remind_SSP2-PkBudg500_2050"] + +.. code-block:: python + + { + # written by brightway + "format": "Ecoinvent XML", + "depends": ["ecoinvent-3.10.1-biosphere"], + "backend": "sqlite", + "number": 43648, + "modified": "2026-08-14T12:09:25.945746", + "processed": "2026-08-14T12:09:56.124243", + "geocollections": ["world"], + "searchable": True, + # written by premise + "premise_version": "2.4.9.1", + "iam_model": "remind", + "pathway": "SSP2-PkBudg500", + "representative_time": "2050-01-01T00:00:00", + "ecoinvent_version": "3.10.1", + "system_model": "cutoff", + } + +.. note:: + + ``bd.databases["db_name"]`` and ``bd.Database("db_name").metadata`` are the same + mapping, so the fields above can be read either way. + +``representative_time`` is the ISO 8601 point in time the database is representative of, +and can be read back with ``datetime.fromisoformat()``. + +Databases holding several scenarios (superstructure and scenario-array databases) +list them under a ``scenarios`` key instead, and only carry ``representative_time`` +at the top level if all their scenarios share the same year. +User (external) scenarios, if any, are listed under ``external_scenarios``. + + Superstructure database *********************** diff --git a/premise/brightway2.py b/premise/brightway2.py index e6f73613..b2e2b4af 100644 --- a/premise/brightway2.py +++ b/premise/brightway2.py @@ -442,11 +442,20 @@ def _compact_payload_for_fast_write(data: list) -> list: return data +def _store_database_metadata(name: str, metadata: dict = None) -> None: + """Attach scenario metadata to a registered Brightway database.""" + if not metadata or name not in databases: + return + databases[name].update(metadata) + databases.flush() + + def write_brightway_database( data: list, name: str, fast: bool = False, check_internal: bool = True, + metadata: dict = None, ) -> None: """ Write a Brightway2 database from a Wurst database. @@ -481,4 +490,5 @@ def write_brightway_database( else: databases[name].pop("geocollections", None) databases.flush() + _store_database_metadata(name, metadata) _print_database_written(name) diff --git a/premise/brightway25.py b/premise/brightway25.py index 603616b6..d6a237b7 100644 --- a/premise/brightway25.py +++ b/premise/brightway25.py @@ -775,11 +775,20 @@ def iter_technosphere(): _write_search_index_fast(db.filename, data, name) +def _store_database_metadata(name: str, metadata: dict = None) -> None: + """Attach scenario metadata to a registered Brightway database.""" + if not metadata or name not in databases: + return + databases[name].update(metadata) + databases.set_modified(name) + + def write_brightway_database( data: list, name: str, fast: bool = False, check_internal: bool = True, + metadata: dict = None, ) -> None: for act in data: act.setdefault("database", name) @@ -802,6 +811,7 @@ def write_brightway_database( _print_database_overwrite(name) _compact_payload_for_fast_write(data, name) _write_processed_database_fast(data, name) + _store_database_metadata(name, metadata) _print_database_written(name) return @@ -811,4 +821,5 @@ def write_brightway_database( with _fast_sqlite_writes(fast): BW25Importer(name, data).write_database() + _store_database_metadata(name, metadata) _print_database_written(name) diff --git a/premise/new_database.py b/premise/new_database.py index 8d725047..370ee42b 100644 --- a/premise/new_database.py +++ b/premise/new_database.py @@ -59,6 +59,7 @@ from .transport import _update_vehicles from .utils import ( cache_ref_exists, + database_metadata, clear_existing_cache, clear_runtime_caches, create_scenario_list, @@ -76,6 +77,7 @@ end_of_process, create_cache, restore_cached_classifications, + scenario_metadata, ) from .renewables import _update_wind_turbines @@ -1291,6 +1293,11 @@ def write_superstructure_db_to_brightway( name=name, fast=True, check_internal=False, + metadata=database_metadata( + self.scenarios, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) self._finalize_superstructure_export() @@ -1359,6 +1366,11 @@ def write_scenario_array_db_to_brightway( name=name, fast=True, check_internal=False, + metadata=database_metadata( + self.scenarios, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) ordered_labels = ["original", *scenario_labels] @@ -1562,6 +1574,11 @@ def write_db_to_brightway(self, name: [str, List[str]] = None): name[s], fast=True, check_internal=True, + metadata=scenario_metadata( + scenario, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) end_of_process(scenario) continue @@ -1591,6 +1608,11 @@ def write_db_to_brightway(self, name: [str, List[str]] = None): write_brightway_database( scenario["database"], name[s], + metadata=scenario_metadata( + scenario, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) end_of_process(scenario) diff --git a/premise/utils.py b/premise/utils.py index 0d304e36..c9751747 100644 --- a/premise/utils.py +++ b/premise/utils.py @@ -160,6 +160,104 @@ def eidb_label( return name +def scenario_metadata( + scenario: Dict[str, Any], + version: str = None, + system_model: str = None, +) -> Dict[str, Any]: + """Describe the scenario a database represents. + + The returned mapping is meant to be attached to the metadata of an exported + database (e.g., ``bw2data.databases[name]``), so that the point in time and + the IAM scenario it represents can be retrieved later on. + + :param scenario: Scenario dictionary, with `model`, `pathway` and `year` keys. + :type scenario: dict + :param version: Ecoinvent version the database is based on. + :type version: str + :param system_model: Ecoinvent system model ("cutoff" or "consequential"). + :type system_model: str + :return: JSON-serializable scenario metadata. + :rtype: dict + """ + + year = int(scenario["year"]) + + metadata = { + "premise_version": ".".join(str(item) for item in __version__), + "iam_model": scenario["model"], + "pathway": scenario["pathway"], + # ISO 8601 point in time the database is representative of + "representative_time": datetime(year, 1, 1).isoformat(), + } + + if version is not None: + metadata["ecoinvent_version"] = str(version) + + if system_model is not None: + metadata["system_model"] = system_model + + external_scenarios = scenario.get("external scenarios") + if external_scenarios: + metadata["external_scenarios"] = [ + ext["scenario"] for ext in external_scenarios if "scenario" in ext + ] + + return metadata + + +def database_metadata( + scenarios: List[Dict[str, Any]], + version: str = None, + system_model: str = None, +) -> Dict[str, Any]: + """Describe the scenario(s) a database represents. + + Databases holding a single scenario get a flat description + (see :func:`scenario_metadata`). Databases holding several scenarios + (super-structure or scenario-array databases) get the list of scenarios + under `scenarios`, plus the point in time they all share, if any. + + :param scenarios: List of scenario dictionaries. + :type scenarios: list + :param version: Ecoinvent version the database is based on. + :type version: str + :param system_model: Ecoinvent system model ("cutoff" or "consequential"). + :type system_model: str + :return: JSON-serializable database metadata. + :rtype: dict + """ + + entries = [ + scenario_metadata(scenario, version=version, system_model=system_model) + for scenario in scenarios + ] + + if len(entries) == 1: + return entries[0] + + for entry in entries: + # reported once, at the database level + entry.pop("premise_version", None) + + metadata = { + "premise_version": ".".join(str(item) for item in __version__), + "scenarios": entries, + } + + if version is not None: + metadata["ecoinvent_version"] = str(version) + + if system_model is not None: + metadata["system_model"] = system_model + + times = {entry["representative_time"] for entry in entries} + if len(times) == 1: + metadata["representative_time"] = times.pop() + + return metadata + + @lru_cache(maxsize=1) def load_constants() -> Dict[str, Any]: """Load global constants from ``constants.yaml``. diff --git a/tests/test_database_metadata.py b/tests/test_database_metadata.py new file mode 100644 index 00000000..cb48f41f --- /dev/null +++ b/tests/test_database_metadata.py @@ -0,0 +1,166 @@ +"""Tests for the scenario metadata attached to exported Brightway databases.""" + +import premise.brightway2 as brightway2_module +import premise.brightway25 as brightway25_module +from premise import __version__ +from premise.utils import database_metadata, scenario_metadata + +SCENARIO = { + "model": "remind", + "pathway": "SSP2-PkBudg500", + "year": 2050, +} + + +def test_scenario_metadata_describes_scenario_and_time(): + metadata = scenario_metadata( + SCENARIO, version="3.10.1", system_model="cutoff" + ) + + assert metadata["iam_model"] == "remind" + assert metadata["pathway"] == "SSP2-PkBudg500" + assert metadata["representative_time"] == "2050-01-01T00:00:00" + # the year is already carried by the ISO timestamp + assert "year" not in metadata + assert metadata["ecoinvent_version"] == "3.10.1" + assert metadata["system_model"] == "cutoff" + assert metadata["premise_version"] == ".".join(str(i) for i in __version__) + assert "external_scenarios" not in metadata + + +def test_scenario_metadata_includes_external_scenarios(): + scenario = dict( + SCENARIO, + **{ + "external scenarios": [ + {"scenario": "Business As Usual", "data": {"some": "package"}} + ] + }, + ) + + metadata = scenario_metadata(scenario) + + assert metadata["external_scenarios"] == ["Business As Usual"] + assert "ecoinvent_version" not in metadata + + +def test_database_metadata_for_single_scenario_is_flat(): + metadata = database_metadata([SCENARIO], version="3.10.1", system_model="cutoff") + + assert metadata == scenario_metadata( + SCENARIO, version="3.10.1", system_model="cutoff" + ) + + +def test_database_metadata_for_several_scenarios_lists_them(): + scenarios = [SCENARIO, dict(SCENARIO, year=2030)] + + metadata = database_metadata(scenarios, version="3.10.1", system_model="cutoff") + + assert [s["representative_time"] for s in metadata["scenarios"]] == [ + "2050-01-01T00:00:00", + "2030-01-01T00:00:00", + ] + assert metadata["ecoinvent_version"] == "3.10.1" + assert metadata["system_model"] == "cutoff" + # years differ, so no single representative point in time + assert "representative_time" not in metadata + + +def test_database_metadata_shares_time_when_scenarios_agree(): + scenarios = [SCENARIO, dict(SCENARIO, pathway="SSP2-NPi")] + + metadata = database_metadata(scenarios) + + assert metadata["representative_time"] == "2050-01-01T00:00:00" + assert len(metadata["scenarios"]) == 2 + + +class DummyDatabases(dict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.flushed = 0 + self.modified = [] + + def flush(self): + self.flushed += 1 + + def set_modified(self, name): + self.modified.append(name) + + +def test_write_brightway2_database_stores_metadata(monkeypatch): + monkeypatch.setattr(brightway2_module, "change_db_name", lambda data, name: None) + monkeypatch.setattr(brightway2_module, "link_internal", lambda data: None) + monkeypatch.setattr(brightway2_module, "check_internal_linking", lambda data: None) + + class DummyImporter: + def __init__(self, name, data): + self.name = name + + def write_database(self): + brightway2_module.databases[self.name] = {} + + monkeypatch.setattr(brightway2_module, "BW2Importer", DummyImporter) + databases = DummyDatabases() + monkeypatch.setattr(brightway2_module, "databases", databases) + + brightway2_module.write_brightway_database( + data=[{"code": "a", "location": "CH", "type": "process", "exchanges": []}], + name="bw2-db", + metadata={"iam_model": "remind", "representative_time": "2050-01-01T00:00:00"}, + ) + + assert databases["bw2-db"]["iam_model"] == "remind" + assert databases["bw2-db"]["representative_time"] == "2050-01-01T00:00:00" + + +def test_write_brightway25_database_stores_metadata_on_fast_path(monkeypatch): + monkeypatch.setattr(brightway25_module, "change_db_name", lambda data, name: None) + monkeypatch.setattr(brightway25_module, "link_internal", lambda data: None) + monkeypatch.setattr(brightway25_module, "check_internal_linking", lambda data: None) + monkeypatch.setattr( + brightway25_module, "_compact_payload_for_fast_write", lambda data, name: None + ) + monkeypatch.setattr( + brightway25_module, "_write_processed_database_fast", lambda data, name: None + ) + databases = DummyDatabases({"fast-db": {}}) + monkeypatch.setattr(brightway25_module, "databases", databases) + + brightway25_module.write_brightway_database( + data=[{"code": "a", "exchanges": []}], + name="fast-db", + fast=True, + metadata={"iam_model": "image", "representative_time": "2030-01-01T00:00:00"}, + ) + + assert databases["fast-db"]["iam_model"] == "image" + assert databases["fast-db"]["representative_time"] == "2030-01-01T00:00:00" + assert databases.modified == ["fast-db"] + + +def test_write_brightway25_database_stores_metadata_on_slow_path(monkeypatch): + monkeypatch.setattr(brightway25_module, "change_db_name", lambda data, name: None) + monkeypatch.setattr(brightway25_module, "link_internal", lambda data: None) + monkeypatch.setattr(brightway25_module, "check_internal_linking", lambda data: None) + + class DummyImporter: + def __init__(self, name, data): + self.name = name + + def write_database(self): + brightway25_module.databases[self.name] = {} + + monkeypatch.setattr(brightway25_module, "BW25Importer", DummyImporter) + databases = DummyDatabases() + monkeypatch.setattr(brightway25_module, "databases", databases) + + brightway25_module.write_brightway_database( + data=[{"code": "a", "exchanges": []}], + name="slow-db", + fast=False, + metadata={"pathway": "SSP2-RCP19"}, + ) + + assert databases["slow-db"]["pathway"] == "SSP2-RCP19" diff --git a/tests/test_new_database.py b/tests/test_new_database.py index e0fd6684..371479f5 100644 --- a/tests/test_new_database.py +++ b/tests/test_new_database.py @@ -236,12 +236,15 @@ def fake_prepare_db_for_fast_export(scenario, name, biosphere_name, version): } return prepared_database - def fake_write_brightway_database(data, name, fast=False, check_internal=True): + def fake_write_brightway_database( + data, name, fast=False, check_internal=True, metadata=None + ): captured["written"] = { "data": data, "name": name, "fast": fast, "check_internal": check_internal, + "metadata": metadata, } monkeypatch.setattr( @@ -314,12 +317,15 @@ def fake_write_brightway_database(data, name, fast=False, check_internal=True): "biosphere_name": "test-biosphere", "version": "3.12", } - assert captured["written"] == { - "data": prepared_database, - "name": "fast-db", - "fast": True, - "check_internal": True, - } + assert captured["written"]["data"] == prepared_database + assert captured["written"]["name"] == "fast-db" + assert captured["written"]["fast"] is True + assert captured["written"]["check_internal"] is True + assert captured["written"]["metadata"]["iam_model"] == "image" + assert captured["written"]["metadata"]["pathway"] == "SSP2-Base" + assert ( + captured["written"]["metadata"]["representative_time"] == "2030-01-01T00:00:00" + ) assert captured["ended"] == [ { "model": "image", @@ -488,12 +494,15 @@ def fake_prepare_db_for_export( } return prepared_database - def fake_write_brightway_database(data, name, fast=False, check_internal=True): + def fake_write_brightway_database( + data, name, fast=False, check_internal=True, metadata=None + ): captured["written"] = { "data": data, "name": name, "fast": fast, "check_internal": check_internal, + "metadata": metadata, } monkeypatch.setattr( @@ -569,12 +578,13 @@ def fake_write_brightway_database(data, name, fast=False, check_internal=True): "biosphere_name": "test-biosphere", "version": "3.12", } - assert captured["written"] == { - "data": prepared_database, - "name": "super-db", - "fast": True, - "check_internal": False, - } + assert captured["written"]["data"] == prepared_database + assert captured["written"]["name"] == "super-db" + assert captured["written"]["fast"] is True + assert captured["written"]["check_internal"] is False + assert [ + s["representative_time"] for s in captured["written"]["metadata"]["scenarios"] + ] == [f"{scenario['year']}-01-01T00:00:00" for scenario in obj.scenarios] assert captured["ended"] == obj.scenarios assert captured["pickles_deleted"] == 1 @@ -745,12 +755,13 @@ def fake_write_package(**kwargs): "delete", ] database_call = events[1][1] - assert database_call == { - "data": prepared_database, - "name": "scenario-db", - "fast": True, - "check_internal": False, - } + assert database_call["data"] == prepared_database + assert database_call["name"] == "scenario-db" + assert database_call["fast"] is True + assert database_call["check_internal"] is False + assert [ + s["representative_time"] for s in database_call["metadata"]["scenarios"] + ] == [f"{scenario['year']}-01-01T00:00:00" for scenario in obj.scenarios] package_call = events[2][1] assert package_call["dataframe"] is dataframe assert package_call["scenario_labels"] == [