Add extra schema checker jobs to CI #577
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
| #!/bin/bash | ||
| # Script to verify that all properties in the specification have corresponding schema definitions | ||
| # Compares fields extracted from optimade.rst with schema definition files | ||
|
|
||
| # Colors for output | ||
| RED='\033[0;31m' | ||
| GREEN='\033[0;32m' | ||
| YELLOW='\033[0;33m' | ||
| NC='\033[0m' # No Color | ||
|
|
||
| # Extract fields from specification | ||
| spec_fields=$(tests/scripts/extract_entry_fields.sh) | ||
|
|
||
| # Track errors | ||
| errors=0 | ||
| warnings=0 | ||
|
|
||
| echo "Checking property definitions against specification..." | ||
| echo "" | ||
|
|
||
| # Function to check if a property definition file exists | ||
| check_property() { | ||
| local entry_type=$1 | ||
| local property=$2 | ||
| local version=${3:-v1.2} # Default to v1.2 | ||
|
Member
There was a problem hiding this comment. Instead of hard-coding, maybe we can extract version numbers (here and below for |
||
|
|
||
| # Check in the main entry type directory first | ||
| local primary_path="schemas/src/defs/$version/properties/optimade/$entry_type/$property.yaml" | ||
|
Member
There was a problem hiding this comment. I think the proper way to check for the existence of a property is:
Important point here is that |
||
|
|
||
| # Also check v1.3 for newer properties | ||
| local v13_path="schemas/src/defs/v1.3/properties/optimade/$entry_type/$property.yaml" | ||
|
|
||
| # Check core properties (shared across types) | ||
| local core_path="schemas/src/defs/$version/properties/core/$property.yaml" | ||
|
|
||
| if [[ -f "$primary_path" ]] || [[ -f "$v13_path" ]] || [[ -f "$core_path" ]]; then | ||
| return 0 | ||
| else | ||
| return 1 | ||
| fi | ||
| } | ||
|
|
||
| # Get the entry types from the JSON output | ||
| entry_types=$(echo "$spec_fields" | jq -r 'keys[]') | ||
|
|
||
| for entry_type in $entry_types; do | ||
| echo "Checking $entry_type..." | ||
|
|
||
| # Special handling for _common (these should be in core/) | ||
| if [[ "$entry_type" == "_common" ]]; then | ||
| properties=$(echo "$spec_fields" | jq -r '.["_common"][]') | ||
| for property in $properties; do | ||
| # Common properties should exist in core/ or be shared | ||
| if [[ -f "schemas/src/defs/v1.2/properties/core/$property.yaml" ]]; then | ||
| echo -e " ${GREEN}✓${NC} $property (core)" | ||
| else | ||
| # Check if it exists in any entry type | ||
| found=false | ||
| for et in $entry_types; do | ||
| if check_property "$et" "$property"; then | ||
| echo -e " ${GREEN}✓${NC} $property (in $et)" | ||
| found=true | ||
| break | ||
| fi | ||
| done | ||
| if [[ "$found" == "false" ]]; then | ||
| echo -e " ${RED}✗${NC} $property - MISSING" | ||
| ((errors++)) | ||
| fi | ||
| fi | ||
| done | ||
| else | ||
| properties=$(echo "$spec_fields" | jq -r --arg et "$entry_type" '.[$et][]') | ||
|
|
||
| # Skip if no properties (empty array) - this is OK for some entry types like calculations | ||
| if [[ -z "$properties" ]]; then | ||
| echo -e " No specific properties with ~~~~ underlines (uses common properties only)" | ||
| continue | ||
| fi | ||
|
|
||
| for property in $properties; do | ||
| if check_property "$entry_type" "$property"; then | ||
| echo -e " ${GREEN}✓${NC} $property" | ||
| else | ||
| # Special case: references properties are defined but not with ~~~~ in spec | ||
| if [[ "$entry_type" == "references" ]]; then | ||
| # References are documented differently, check if the file exists anyway | ||
| if [[ -f "schemas/src/defs/v1.2/properties/optimade/references/$property.yaml" ]]; then | ||
| echo -e " ${GREEN}✓${NC} $property" | ||
| else | ||
| echo -e " ${RED}✗${NC} $property - MISSING" | ||
| ((errors++)) | ||
| fi | ||
| else | ||
| echo -e " ${RED}✗${NC} $property - MISSING" | ||
| ((errors++)) | ||
| fi | ||
| fi | ||
| done | ||
| fi | ||
| echo "" | ||
| done | ||
|
|
||
| # Also check for orphaned schema files (schemas without spec entries) | ||
| echo "Checking for properties defined in schemas but not in specification..." | ||
| echo "" | ||
|
|
||
| for entry_type in structures files references calculations trajectories; do | ||
| schema_dir="schemas/src/defs/v1.2/properties/optimade/$entry_type" | ||
|
|
||
| if [[ ! -d "$schema_dir" ]]; then | ||
| # Check v1.3 | ||
| schema_dir="schemas/src/defs/v1.3/properties/optimade/$entry_type" | ||
| if [[ ! -d "$schema_dir" ]]; then | ||
| continue | ||
| fi | ||
| fi | ||
|
|
||
| # Get properties from spec for this entry type | ||
| if [[ "$entry_type" == "trajectories" ]]; then | ||
| spec_props=$(echo "$spec_fields" | jq -r '.trajectories[]?' 2>/dev/null || echo "") | ||
| else | ||
| spec_props=$(echo "$spec_fields" | jq -r --arg et "$entry_type" '.[$et][]?' 2>/dev/null || echo "") | ||
| fi | ||
|
|
||
| # Also include common properties | ||
| common_props=$(echo "$spec_fields" | jq -r '._common[]?' 2>/dev/null || echo "") | ||
| all_spec_props=$(echo -e "$spec_props\n$common_props" | sort | uniq) | ||
|
|
||
| # Find all schema files | ||
| for schema_file in "$schema_dir"/*.yaml; do | ||
| if [[ ! -f "$schema_file" ]]; then | ||
| continue | ||
| fi | ||
|
|
||
| property=$(basename "$schema_file" .yaml) | ||
|
|
||
| # Check if this property is in the spec | ||
| if echo "$all_spec_props" | grep -q "^${property}$"; then | ||
| : # Found, nothing to do | ||
| else | ||
| if [[ "$entry_type" == "references" ]]; then | ||
| # References properties might not be in spec with ~~~~ but still valid | ||
| if grep -q "^${property}$" <<< "$spec_props"; then | ||
| echo -e " ${GREEN}✓${NC} $entry_type/$property" | ||
| continue | ||
| fi | ||
| else | ||
| echo -e " ${YELLOW}⚠${NC} $entry_type/$property - Schema exists but not in specification (or uses different formatting)" | ||
| ((warnings++)) | ||
| fi | ||
| fi | ||
| done | ||
| done | ||
|
|
||
| echo "" | ||
| echo "==============================================" | ||
| echo "Summary:" | ||
| echo " Errors: $errors" | ||
| echo " Warnings: $warnings" | ||
| echo "==============================================" | ||
|
|
||
| if [[ $errors -gt 0 ]]; then | ||
| exit 1 | ||
| else | ||
| exit 0 | ||
| fi | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| #!/bin/bash | ||
| # Script to extract field names from optimade.rst Entry List section | ||
| # Extracts properties with ~~~~ underlines and groups by entry type | ||
|
|
||
| input_file="optimade.rst" | ||
|
|
||
| if [[ ! -f "$input_file" ]]; then | ||
| echo "Error: $input_file not found" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Use awk to process the entire file | ||
| awk ' | ||
| BEGIN { | ||
| in_entry_list = 0 | ||
| in_section = 0 | ||
| current_section = "" | ||
| prev_line = "" | ||
| print "{" | ||
| first_section = 1 | ||
| } | ||
|
|
||
| # Detect "Entry List" section start | ||
| /^Entry List$/ { | ||
| getline # Read the ===== line | ||
| in_entry_list = 1 | ||
| next | ||
| } | ||
|
|
||
| # Exit Entry List section when we hit another major section (with ======) | ||
| in_entry_list && /^=+$/ && prev_line !~ /^Entry List$/ { | ||
| in_entry_list = 0 | ||
| } | ||
|
|
||
| # Detect "Properties Used by Multiple Entry Types" | ||
| in_entry_list && /^Properties Used by Multiple Entry Types$/ { | ||
| if (in_section) { | ||
| print " ]," | ||
| } else if (!first_section) { | ||
| print "," | ||
| } | ||
| print " \"_common\": [" | ||
| current_section = "_common" | ||
| in_section = 1 | ||
| first_section = 0 | ||
| first_field = 1 | ||
| next | ||
| } | ||
|
|
||
| # Detect entry type sections (e.g., "Structures Entries") | ||
| in_entry_list && /^[A-Z][a-z]+ Entries$/ { | ||
| if (in_section) { | ||
| print "" | ||
| print " ]," | ||
| } else if (!first_section) { | ||
| print "," | ||
| } | ||
| # Extract entry type name and convert to lowercase | ||
| entry_type = tolower($1) | ||
| printf " \"%s\": [\n", entry_type | ||
| current_section = entry_type | ||
| in_section = 1 | ||
| first_section = 0 | ||
| first_field = 1 | ||
| next | ||
| } | ||
|
|
||
| # Detect field names (lines with ~~~~ underneath) | ||
| in_section && /^~+$/ && prev_line ~ /^[a-z_][a-z_0-9\\]*$/ { | ||
| # prev_line contains the field name | ||
| if (!first_field) { | ||
| print "," | ||
| } | ||
| # Handle escaped underscores in field names | ||
| field = prev_line | ||
| gsub(/\\/, "", field) # Remove backslashes | ||
| printf " \"%s\"", field | ||
| first_field = 0 | ||
| } | ||
|
|
||
| # Store current line for next iteration | ||
| { | ||
| prev_line = $0 | ||
| } | ||
|
|
||
| END { | ||
| if (in_section) { | ||
| print "" | ||
| print " ]" | ||
| } | ||
| print "}" | ||
| } | ||
| ' "$input_file" |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.