Skip to content

Update automatic metadata creation #393

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
chrwm wants to merge 12 commits into
base: develop
Choose a base branch
from
Draft

Update automatic metadata creation #393

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
a63e962
Update link webdienst #386
Dec 1, 2022
dc076c5
Extend logging #386
Dec 1, 2022
6078539
Update static metadata #386
Dec 1, 2022
e0b63db
Merge branch 'develop' into feature-386-update-auto-metadata-creation
Mar 6, 2023
8e9f66e
Update metadataversion to 1.5.2.#386
Mar 16, 2023
3ebf1bb
Update metadata in datapackage_meta to v 1.5.2.#386
Mar 21, 2023
dd0aa8e
Update contributors to v 1.5.2.#386
Mar 28, 2023
b98761a
Update resources to v 1.5.2.#386
Mar 28, 2023
7cd6690
Update resources to v 1.5.2.#386
Mar 28, 2023
1b96672
Fix error in resources #386
Mar 29, 2023
6b73afb
Fix typo in processed fields #386
Mar 29, 2023
2c78a04
Fix format key in resources #386
Mar 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions open_mastr/mastr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def __init__(self, engine="sqlite") -> None:

self.engine = create_database_engine(engine, self.home_directory)

print(
log.info(
f"Data will be written to the following database: {self.engine.url}\n"
"If you run into problems, try to "
"delete the database and update the package by running 'pip install --upgrade open-mastr'\n"
Expand Down Expand Up @@ -236,7 +236,7 @@ def download(
# Set api_processes to None in order to avoid the malfunctioning usage
if api_processes:
api_processes = None
print(
log.info(
"Warning: The implementation of parallel processes is currently under construction. Please let "
"the argument api_processes at the default value None."
)
Expand Down Expand Up @@ -353,7 +353,7 @@ def to_csv(
try:
df = pd.read_sql(additional_table, con=self.engine)
except ValueError as e:
print(
log.info(
f"While reading table '{additional_table}', the following error occured: {e}"
)
continue
Expand Down
70 changes: 46 additions & 24 deletions open_mastr/soap_api/metadata/description.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,19 @@ def __init__(self, xml=None):
self.xml = fh.read()
else:
# If no XML file is given, the file is read from an URL
zipurl = 'https://www.marktstammdatenregister.de/MaStRHilfe/files/' \
'webdienst/Dienstbeschreibung_1_2_39_Produktion.zip'
zipurl = "https://www.marktstammdatenregister.de/MaStRHilfe/files/webdienst/" \
"Dienstbeschreibung_Produktion_Version" \
"1.2.87" \ # update version here
".zip"

with urlopen(zipurl) as zipresp:
with ZipFile(BytesIO(zipresp.read())) as zfile:
self.xml = zfile.read('xsd/mastrbasetypes.xsd')


self.xml = zfile.read("xsd/mastrbasetypes.xsd")

# Parse XML and extract relevant data
parsed = xmltodict.parse(self.xml, process_namespaces=False)
self.complex_types = parsed['schema']["complexType"]
self.simple_types = parsed['schema']["simpleType"]
self.complex_types = parsed["schema"]["complexType"]
self.simple_types = parsed["schema"]["simpleType"]

# Prepare parsed data for documentational purposes
abstract_types, parameters, responses, types = self._filter_type_descriptions()
Expand Down Expand Up @@ -78,13 +78,17 @@ def _filter_type_descriptions(self):
raise ValueError("Ohh...")
else:
# Filter all functions
if item["@name"].startswith(("Get", "Set", "Erneute", "Verschiebe", "Delete")):
if item["@name"].startswith(
("Get", "Set", "Erneute", "Verschiebe", "Delete")
):
functions.append(item)

# Further split the list of functions into paramters and responses
if item["@name"].endswith("Parameter"):
if "complexContent" in item.keys():
parameters[item["@name"]] = item["complexContent"]["extension"]
parameters[item["@name"]] = item["complexContent"][
"extension"
]
else:
parameters[item["@name"]] = item
elif item["@name"].endswith("Antwort"):
Expand All @@ -111,12 +115,14 @@ def prepare_simple_type(self):

for simple_type in self.simple_types:
if "enumeration" in simple_type["restriction"]:
possible_values = [_["@value"] for _ in simple_type["restriction"]["enumeration"]]
possible_values = [
_["@value"] for _ in simple_type["restriction"]["enumeration"]
]
else:
possible_values = []
simple_types_doc[simple_type["@name"]] = {
"type": simple_type["restriction"]["@base"],
"values": possible_values
"values": possible_values,
}
return simple_types_doc

Expand All @@ -140,49 +146,61 @@ def functions_data_documentation(self):
if "annotation" in fcn["sequence"]["element"]:
fcn_data = [fcn["sequence"]["element"]]
else:
fcn_data = self.types[fcn["sequence"]["element"]["@type"].split(":")[1]]["sequence"]["element"]
fcn_data = self.types[
fcn["sequence"]["element"]["@type"].split(":")[1]
]["sequence"]["element"]
else:
print(type(fcn["sequence"]))
print(fcn["sequence"])
raise ValueError

# Add data for inherited columns from base types
if "@base" in fcn:
if not fcn["@base"] == 'mastr:AntwortBasis':
fcn_data = _collect_columns_of_base_type(self.types, fcn["@base"].split(":")[1], fcn_data)
if not fcn["@base"] == "mastr:AntwortBasis":
fcn_data = _collect_columns_of_base_type(
self.types, fcn["@base"].split(":")[1], fcn_data
)
function_docs[fcn_name] = {}
for column in fcn_data:
# Replace MaStR internal types with more general ones
if column["@type"].startswith("mastr:"):
try:
column_type = self.simple_types_prepared[column["@type"].split(":")[1]]["type"]
column_type = self.simple_types_prepared[
column["@type"].split(":")[1]
]["type"]
except KeyError:
column_type = column["@type"]
else:
column_type = column["@type"]

if "annotation" in column.keys():
description = column["annotation"]["documentation"].get("#text", None)
description = column["annotation"]["documentation"].get(
"#text", None
)
if description:
description = re.sub(" +", " ", description.replace("\n", ""))
description = re.sub(
" +", " ", description.replace("\n", "")
)
function_docs[fcn_name][column["@name"]] = {
"type": column_type,
"description": description,
"example": column["annotation"]["documentation"].get("m-ex", None)
"type": column_type,
"description": description,
"example": column["annotation"]["documentation"].get(
"m-ex", None
),
}
else:
function_docs[fcn_name][column["@name"]] = {
"type": column_type,
# TODO: insert information from simple type here
"description": None,
"example": None
"example": None,
}

# Hack in a descrition for a column that gets created after download while flattening data
function_docs["GetEinheitWind"]["HerstellerId"] = {
"type": "str",
"description": "Id des Herstellers der Einheit",
"example": 923
"example": 923,
}

return function_docs
Expand All @@ -193,7 +211,11 @@ def _collect_columns_of_base_type(base_types, base_type_name, fcn_data):
fcn_data += type_description["extension"]["sequence"]["element"]

if "@base" in type_description["extension"]:
if not type_description["extension"]["@base"] == 'mastr:AntwortBasis':
fcn_data = _collect_columns_of_base_type(base_types, type_description["extension"]["@base"].split(":")[1], fcn_data)
if not type_description["extension"]["@base"] == "mastr:AntwortBasis":
fcn_data = _collect_columns_of_base_type(
base_types,
type_description["extension"]["@base"].split(":")[1],
fcn_data,
)

return fcn_data
65 changes: 43 additions & 22 deletions open_mastr/soap_api/metadata/mastr_datapackage.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,29 @@
],
"subject": [
{
"name": null,
"path": null
"name": "power plant",
"path": "http://openenergy-platform.org/ontology/oeo/OEO_00000031"

@deniztepe deniztepe Dec 8, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the function of ontology references?

@chrwm chrwm Dec 8, 2022 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The purpose is that the @context field makes some fields in the oemetadata searchable via sparql queries. Amongst others subject, isAbout, valueReference. It makes the metadata JSON to a JSON-LD.
The idea is that in the future datasets are annotated ontologically for two use-cases:

  1. common vocabulary to search for data in the energy domain in a search engine that builds upon linked open data
    Here the prototype of a search engine from LOD-GEOSS project: http://moss.tools.dbpedia.org/search
    The usability and visualistion are going to be improved.
    When you search for hub height, select the concept from the suggestions and search for it on the energy databus, you'll find wind power plant datasets that contain information about hub height you might use for your calculations. The datasets are registered in the databus, which is a metadata catalog that points to decentrally hosted databases. So the data is not actually hosted and maintained on the databus but at individual institutions with the idea to improve data findability in the domain (only works if sufficient datapoints with good metadata participate).
  2. inference of knowlegde with annotated datasets
    simplistic example: I annote the mastr dataset with power plant and you know generally what the concept power plant is, but not what types of power plants exist in the energy domain. You could gain knowledge from the information stored via the hierarchical relations in the ontology.
    https://openenergy-platform.org/viewer/oeo/ if you search for power plant here.

},
{
"name": "renewable",
"path": "http://openenergy-platform.org/ontology/oeo/OEO_00030004"
},
{
"name": "conventional",
"path": "http://openenergy-platform.org/ontology/oeo/OEO_00020147"
},
{
"name": "energy storage object",
"path": "http://openenergy-platform.org/ontology/oeo/OEO_00000159"
}
],
"keywords": [
"powerplants",
"renewables"
"renewables",
"coventional",
"storage"
],
"publicationDate": "2022-05-16",
"publicationDate": "2022-12-01",
"context": {
"homepage": "https://www.marktstammdatenregister.de/MaStR/",
"documentation": "https://www.marktstammdatenregister.de/MaStRHilfe/index.html",
Expand All @@ -34,7 +48,7 @@
"resolution": "vector"
},
"temporal": {
"referenceDate": "2022-05-16",
"referenceDate": "2022-12-01",
"timeseries": [
{
"start": null,
Expand Down Expand Up @@ -154,13 +168,20 @@
"date": "2022-05-16",
"object": "metadata and data",
"comment": "Update metadata and run download"
},
{
"title": "chrwm",
"email": null,
"date": "2022-12-01",
"object": "metadata and data",
"comment": "Update metadata and run bulk download with open-MaStR v0.12.2"
}
],
"resources": [
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_wind",
"path": "bnetza_open_mastr_wind.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -3130,14 +3151,14 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
},
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_hydro",
"path": "bnetza_open_mastr_hydro.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -5651,14 +5672,14 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
},
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_biomass",
"path": "bnetza_open_mastr_biomass.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -8381,14 +8402,14 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
},
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_solar",
"path": "bnetza_open_mastr_solar.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -11187,14 +11208,14 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
},
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_storage",
"path": "bnetza_open_mastr_storage.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -13556,14 +13577,14 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
},
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_combustion",
"path": "bnetza_open_mastr_combustion.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -16034,14 +16055,14 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
},
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_nuclear",
"path": "bnetza_open_mastr_nuclear.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -17961,14 +17982,14 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
},
{
"profile": "tabular-data-resource",
"name": "bnetza_open_mastr_gsgk",
"path": "bnetza_open_mastr_gsgk.csv",
"path": "https://doi.org/10.5281/zenodo.7387843",
"format": "csv",
"encoding": "UTF-8",
"schema": {
Expand Down Expand Up @@ -20268,13 +20289,13 @@
]
},
"dialect": {
"delimiter": ";",
"delimiter": ",",
"decimalSeparator": "."
}
}
],
"@id": null,
"@context": null,
"@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/master/metadata/latest/context.json",
"review": {
"path": null,
"badge": null
Expand Down
1 change: 1 addition & 0 deletions open_mastr/soap_api/mirror.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1258,6 +1258,7 @@ def to_csv(
)
metadata = datapackage_meta_json(newest_date, technology, json_serialize=False)

log.info("Save metadata")
with open(metadata_file, "w", encoding="utf-8") as f:
json.dump(metadata, f, ensure_ascii=False, indent=4)

Expand Down
Loading