diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b6d9ce..70591298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project are documented in this file. +## [Unreleased] + +### Added +- Databases exported to Brightway now carry scenario metadata in + `bw2data.databases[name]`: `iam_model`, `pathway`, `representative_time` + (ISO 8601), `ecoinvent_version`, `system_model`, `premise_version` and, + if any, `external_scenarios`. Superstructure and scenario-array databases + list their scenarios under `scenarios`. + ## [2.4.9.2] ### Added diff --git a/docs/load.rst b/docs/load.rst index ae5d9912..cec296bd 100644 --- a/docs/load.rst +++ b/docs/load.rst @@ -27,6 +27,54 @@ If several databases have been built, the user can give them specific names, lik biosphere database already registered. +Scenario metadata +***************** + +Every database written to brightway carries, in its brightway metadata, a description +of the scenario and of the point in time it represents. It sits next to the fields +brightway maintains itself: + +.. code-block:: python + + import bw2data as bd + + bd.databases["ei_cutoff_3.10.1_remind_SSP2-PkBudg500_2050"] + +.. code-block:: python + + { + # written by brightway + "format": "Ecoinvent XML", + "depends": ["ecoinvent-3.10.1-biosphere"], + "backend": "sqlite", + "number": 43648, + "modified": "2026-08-14T12:09:25.945746", + "processed": "2026-08-14T12:09:56.124243", + "geocollections": ["world"], + "searchable": True, + # written by premise + "premise_version": "2.4.9.1", + "iam_model": "remind", + "pathway": "SSP2-PkBudg500", + "representative_time": "2050-01-01T00:00:00", + "ecoinvent_version": "3.10.1", + "system_model": "cutoff", + } + +.. note:: + + ``bd.databases["db_name"]`` and ``bd.Database("db_name").metadata`` are the same + mapping, so the fields above can be read either way. + +``representative_time`` is the ISO 8601 point in time the database is representative of, +and can be read back with ``datetime.fromisoformat()``. + +Databases holding several scenarios (superstructure and scenario-array databases) +list them under a ``scenarios`` key instead, and only carry ``representative_time`` +at the top level if all their scenarios share the same year. +User (external) scenarios, if any, are listed under ``external_scenarios``. + + Superstructure database *********************** diff --git a/premise/brightway2.py b/premise/brightway2.py index e6f73613..b2e2b4af 100644 --- a/premise/brightway2.py +++ b/premise/brightway2.py @@ -442,11 +442,20 @@ def _compact_payload_for_fast_write(data: list) -> list: return data +def _store_database_metadata(name: str, metadata: dict = None) -> None: + """Attach scenario metadata to a registered Brightway database.""" + if not metadata or name not in databases: + return + databases[name].update(metadata) + databases.flush() + + def write_brightway_database( data: list, name: str, fast: bool = False, check_internal: bool = True, + metadata: dict = None, ) -> None: """ Write a Brightway2 database from a Wurst database. @@ -481,4 +490,5 @@ def write_brightway_database( else: databases[name].pop("geocollections", None) databases.flush() + _store_database_metadata(name, metadata) _print_database_written(name) diff --git a/premise/brightway25.py b/premise/brightway25.py index 603616b6..d6a237b7 100644 --- a/premise/brightway25.py +++ b/premise/brightway25.py @@ -775,11 +775,20 @@ def iter_technosphere(): _write_search_index_fast(db.filename, data, name) +def _store_database_metadata(name: str, metadata: dict = None) -> None: + """Attach scenario metadata to a registered Brightway database.""" + if not metadata or name not in databases: + return + databases[name].update(metadata) + databases.set_modified(name) + + def write_brightway_database( data: list, name: str, fast: bool = False, check_internal: bool = True, + metadata: dict = None, ) -> None: for act in data: act.setdefault("database", name) @@ -802,6 +811,7 @@ def write_brightway_database( _print_database_overwrite(name) _compact_payload_for_fast_write(data, name) _write_processed_database_fast(data, name) + _store_database_metadata(name, metadata) _print_database_written(name) return @@ -811,4 +821,5 @@ def write_brightway_database( with _fast_sqlite_writes(fast): BW25Importer(name, data).write_database() + _store_database_metadata(name, metadata) _print_database_written(name) diff --git a/premise/new_database.py b/premise/new_database.py index 8d725047..370ee42b 100644 --- a/premise/new_database.py +++ b/premise/new_database.py @@ -59,6 +59,7 @@ from .transport import _update_vehicles from .utils import ( cache_ref_exists, + database_metadata, clear_existing_cache, clear_runtime_caches, create_scenario_list, @@ -76,6 +77,7 @@ end_of_process, create_cache, restore_cached_classifications, + scenario_metadata, ) from .renewables import _update_wind_turbines @@ -1291,6 +1293,11 @@ def write_superstructure_db_to_brightway( name=name, fast=True, check_internal=False, + metadata=database_metadata( + self.scenarios, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) self._finalize_superstructure_export() @@ -1359,6 +1366,11 @@ def write_scenario_array_db_to_brightway( name=name, fast=True, check_internal=False, + metadata=database_metadata( + self.scenarios, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) ordered_labels = ["original", *scenario_labels] @@ -1562,6 +1574,11 @@ def write_db_to_brightway(self, name: [str, List[str]] = None): name[s], fast=True, check_internal=True, + metadata=scenario_metadata( + scenario, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) end_of_process(scenario) continue @@ -1591,6 +1608,11 @@ def write_db_to_brightway(self, name: [str, List[str]] = None): write_brightway_database( scenario["database"], name[s], + metadata=scenario_metadata( + scenario, + version=getattr(self, "version", None), + system_model=getattr(self, "system_model", None), + ), ) end_of_process(scenario) diff --git a/premise/utils.py b/premise/utils.py index 0d304e36..c9751747 100644 --- a/premise/utils.py +++ b/premise/utils.py @@ -160,6 +160,104 @@ def eidb_label( return name +def scenario_metadata( + scenario: Dict[str, Any], + version: str = None, + system_model: str = None, +) -> Dict[str, Any]: + """Describe the scenario a database represents. + + The returned mapping is meant to be attached to the metadata of an exported + database (e.g., ``bw2data.databases[name]``), so that the point in time and + the IAM scenario it represents can be retrieved later on. + + :param scenario: Scenario dictionary, with `model`, `pathway` and `year` keys. + :type scenario: dict + :param version: Ecoinvent version the database is based on. + :type version: str + :param system_model: Ecoinvent system model ("cutoff" or "consequential"). + :type system_model: str + :return: JSON-serializable scenario metadata. + :rtype: dict + """ + + year = int(scenario["year"]) + + metadata = { + "premise_version": ".".join(str(item) for item in __version__), + "iam_model": scenario["model"], + "pathway": scenario["pathway"], + # ISO 8601 point in time the database is representative of + "representative_time": datetime(year, 1, 1).isoformat(), + } + + if version is not None: + metadata["ecoinvent_version"] = str(version) + + if system_model is not None: + metadata["system_model"] = system_model + + external_scenarios = scenario.get("external scenarios") + if external_scenarios: + metadata["external_scenarios"] = [ + ext["scenario"] for ext in external_scenarios if "scenario" in ext + ] + + return metadata + + +def database_metadata( + scenarios: List[Dict[str, Any]], + version: str = None, + system_model: str = None, +) -> Dict[str, Any]: + """Describe the scenario(s) a database represents. + + Databases holding a single scenario get a flat description + (see :func:`scenario_metadata`). Databases holding several scenarios + (super-structure or scenario-array databases) get the list of scenarios + under `scenarios`, plus the point in time they all share, if any. + + :param scenarios: List of scenario dictionaries. + :type scenarios: list + :param version: Ecoinvent version the database is based on. + :type version: str + :param system_model: Ecoinvent system model ("cutoff" or "consequential"). + :type system_model: str + :return: JSON-serializable database metadata. + :rtype: dict + """ + + entries = [ + scenario_metadata(scenario, version=version, system_model=system_model) + for scenario in scenarios + ] + + if len(entries) == 1: + return entries[0] + + for entry in entries: + # reported once, at the database level + entry.pop("premise_version", None) + + metadata = { + "premise_version": ".".join(str(item) for item in __version__), + "scenarios": entries, + } + + if version is not None: + metadata["ecoinvent_version"] = str(version) + + if system_model is not None: + metadata["system_model"] = system_model + + times = {entry["representative_time"] for entry in entries} + if len(times) == 1: + metadata["representative_time"] = times.pop() + + return metadata + + @lru_cache(maxsize=1) def load_constants() -> Dict[str, Any]: """Load global constants from ``constants.yaml``. diff --git a/tests/test_database_metadata.py b/tests/test_database_metadata.py new file mode 100644 index 00000000..cb48f41f --- /dev/null +++ b/tests/test_database_metadata.py @@ -0,0 +1,166 @@ +"""Tests for the scenario metadata attached to exported Brightway databases.""" + +import premise.brightway2 as brightway2_module +import premise.brightway25 as brightway25_module +from premise import __version__ +from premise.utils import database_metadata, scenario_metadata + +SCENARIO = { + "model": "remind", + "pathway": "SSP2-PkBudg500", + "year": 2050, +} + + +def test_scenario_metadata_describes_scenario_and_time(): + metadata = scenario_metadata( + SCENARIO, version="3.10.1", system_model="cutoff" + ) + + assert metadata["iam_model"] == "remind" + assert metadata["pathway"] == "SSP2-PkBudg500" + assert metadata["representative_time"] == "2050-01-01T00:00:00" + # the year is already carried by the ISO timestamp + assert "year" not in metadata + assert metadata["ecoinvent_version"] == "3.10.1" + assert metadata["system_model"] == "cutoff" + assert metadata["premise_version"] == ".".join(str(i) for i in __version__) + assert "external_scenarios" not in metadata + + +def test_scenario_metadata_includes_external_scenarios(): + scenario = dict( + SCENARIO, + **{ + "external scenarios": [ + {"scenario": "Business As Usual", "data": {"some": "package"}} + ] + }, + ) + + metadata = scenario_metadata(scenario) + + assert metadata["external_scenarios"] == ["Business As Usual"] + assert "ecoinvent_version" not in metadata + + +def test_database_metadata_for_single_scenario_is_flat(): + metadata = database_metadata([SCENARIO], version="3.10.1", system_model="cutoff") + + assert metadata == scenario_metadata( + SCENARIO, version="3.10.1", system_model="cutoff" + ) + + +def test_database_metadata_for_several_scenarios_lists_them(): + scenarios = [SCENARIO, dict(SCENARIO, year=2030)] + + metadata = database_metadata(scenarios, version="3.10.1", system_model="cutoff") + + assert [s["representative_time"] for s in metadata["scenarios"]] == [ + "2050-01-01T00:00:00", + "2030-01-01T00:00:00", + ] + assert metadata["ecoinvent_version"] == "3.10.1" + assert metadata["system_model"] == "cutoff" + # years differ, so no single representative point in time + assert "representative_time" not in metadata + + +def test_database_metadata_shares_time_when_scenarios_agree(): + scenarios = [SCENARIO, dict(SCENARIO, pathway="SSP2-NPi")] + + metadata = database_metadata(scenarios) + + assert metadata["representative_time"] == "2050-01-01T00:00:00" + assert len(metadata["scenarios"]) == 2 + + +class DummyDatabases(dict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.flushed = 0 + self.modified = [] + + def flush(self): + self.flushed += 1 + + def set_modified(self, name): + self.modified.append(name) + + +def test_write_brightway2_database_stores_metadata(monkeypatch): + monkeypatch.setattr(brightway2_module, "change_db_name", lambda data, name: None) + monkeypatch.setattr(brightway2_module, "link_internal", lambda data: None) + monkeypatch.setattr(brightway2_module, "check_internal_linking", lambda data: None) + + class DummyImporter: + def __init__(self, name, data): + self.name = name + + def write_database(self): + brightway2_module.databases[self.name] = {} + + monkeypatch.setattr(brightway2_module, "BW2Importer", DummyImporter) + databases = DummyDatabases() + monkeypatch.setattr(brightway2_module, "databases", databases) + + brightway2_module.write_brightway_database( + data=[{"code": "a", "location": "CH", "type": "process", "exchanges": []}], + name="bw2-db", + metadata={"iam_model": "remind", "representative_time": "2050-01-01T00:00:00"}, + ) + + assert databases["bw2-db"]["iam_model"] == "remind" + assert databases["bw2-db"]["representative_time"] == "2050-01-01T00:00:00" + + +def test_write_brightway25_database_stores_metadata_on_fast_path(monkeypatch): + monkeypatch.setattr(brightway25_module, "change_db_name", lambda data, name: None) + monkeypatch.setattr(brightway25_module, "link_internal", lambda data: None) + monkeypatch.setattr(brightway25_module, "check_internal_linking", lambda data: None) + monkeypatch.setattr( + brightway25_module, "_compact_payload_for_fast_write", lambda data, name: None + ) + monkeypatch.setattr( + brightway25_module, "_write_processed_database_fast", lambda data, name: None + ) + databases = DummyDatabases({"fast-db": {}}) + monkeypatch.setattr(brightway25_module, "databases", databases) + + brightway25_module.write_brightway_database( + data=[{"code": "a", "exchanges": []}], + name="fast-db", + fast=True, + metadata={"iam_model": "image", "representative_time": "2030-01-01T00:00:00"}, + ) + + assert databases["fast-db"]["iam_model"] == "image" + assert databases["fast-db"]["representative_time"] == "2030-01-01T00:00:00" + assert databases.modified == ["fast-db"] + + +def test_write_brightway25_database_stores_metadata_on_slow_path(monkeypatch): + monkeypatch.setattr(brightway25_module, "change_db_name", lambda data, name: None) + monkeypatch.setattr(brightway25_module, "link_internal", lambda data: None) + monkeypatch.setattr(brightway25_module, "check_internal_linking", lambda data: None) + + class DummyImporter: + def __init__(self, name, data): + self.name = name + + def write_database(self): + brightway25_module.databases[self.name] = {} + + monkeypatch.setattr(brightway25_module, "BW25Importer", DummyImporter) + databases = DummyDatabases() + monkeypatch.setattr(brightway25_module, "databases", databases) + + brightway25_module.write_brightway_database( + data=[{"code": "a", "exchanges": []}], + name="slow-db", + fast=False, + metadata={"pathway": "SSP2-RCP19"}, + ) + + assert databases["slow-db"]["pathway"] == "SSP2-RCP19" diff --git a/tests/test_new_database.py b/tests/test_new_database.py index e0fd6684..371479f5 100644 --- a/tests/test_new_database.py +++ b/tests/test_new_database.py @@ -236,12 +236,15 @@ def fake_prepare_db_for_fast_export(scenario, name, biosphere_name, version): } return prepared_database - def fake_write_brightway_database(data, name, fast=False, check_internal=True): + def fake_write_brightway_database( + data, name, fast=False, check_internal=True, metadata=None + ): captured["written"] = { "data": data, "name": name, "fast": fast, "check_internal": check_internal, + "metadata": metadata, } monkeypatch.setattr( @@ -314,12 +317,15 @@ def fake_write_brightway_database(data, name, fast=False, check_internal=True): "biosphere_name": "test-biosphere", "version": "3.12", } - assert captured["written"] == { - "data": prepared_database, - "name": "fast-db", - "fast": True, - "check_internal": True, - } + assert captured["written"]["data"] == prepared_database + assert captured["written"]["name"] == "fast-db" + assert captured["written"]["fast"] is True + assert captured["written"]["check_internal"] is True + assert captured["written"]["metadata"]["iam_model"] == "image" + assert captured["written"]["metadata"]["pathway"] == "SSP2-Base" + assert ( + captured["written"]["metadata"]["representative_time"] == "2030-01-01T00:00:00" + ) assert captured["ended"] == [ { "model": "image", @@ -488,12 +494,15 @@ def fake_prepare_db_for_export( } return prepared_database - def fake_write_brightway_database(data, name, fast=False, check_internal=True): + def fake_write_brightway_database( + data, name, fast=False, check_internal=True, metadata=None + ): captured["written"] = { "data": data, "name": name, "fast": fast, "check_internal": check_internal, + "metadata": metadata, } monkeypatch.setattr( @@ -569,12 +578,13 @@ def fake_write_brightway_database(data, name, fast=False, check_internal=True): "biosphere_name": "test-biosphere", "version": "3.12", } - assert captured["written"] == { - "data": prepared_database, - "name": "super-db", - "fast": True, - "check_internal": False, - } + assert captured["written"]["data"] == prepared_database + assert captured["written"]["name"] == "super-db" + assert captured["written"]["fast"] is True + assert captured["written"]["check_internal"] is False + assert [ + s["representative_time"] for s in captured["written"]["metadata"]["scenarios"] + ] == [f"{scenario['year']}-01-01T00:00:00" for scenario in obj.scenarios] assert captured["ended"] == obj.scenarios assert captured["pickles_deleted"] == 1 @@ -745,12 +755,13 @@ def fake_write_package(**kwargs): "delete", ] database_call = events[1][1] - assert database_call == { - "data": prepared_database, - "name": "scenario-db", - "fast": True, - "check_internal": False, - } + assert database_call["data"] == prepared_database + assert database_call["name"] == "scenario-db" + assert database_call["fast"] is True + assert database_call["check_internal"] is False + assert [ + s["representative_time"] for s in database_call["metadata"]["scenarios"] + ] == [f"{scenario['year']}-01-01T00:00:00" for scenario in obj.scenarios] package_call = events[2][1] assert package_call["dataframe"] is dataframe assert package_call["scenario_labels"] == [