Remove strict jsonchemas and conformance test cases (v2)

conformance/ome_zarr_conformance.py

@@ -8,23 +8,73 @@
 """
 
 from __future__ import annotations
-from concurrent.futures import Future, ThreadPoolExecutor
+
+import json
+import logging
 import os
+import re
 import subprocess as sp
-from argparse import ArgumentParser
-from pathlib import Path
 import sys
-import re
-import json
+from argparse import ArgumentParser
+from collections.abc import Iterable
+from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
-from typing import Any, Iterable, Literal, Self
-import logging
+from pathlib import Path
+from typing import Any, Literal, Self
 
 logger = logging.getLogger("ome_zarr_conformance")
 
 here = Path(__file__).resolve().parent.parent
 tests_dir = here / "tests"
 
+Status = Literal["pass", "fail", "error"]
+
+
+def color(s: str, num: int, is_bright=False, is_background=False) -> str:
+    match (is_bright, is_background):
+        case (False, False):
+            pref = "3"
+        case (False, True):
+            pref = "4"
+        case (True, False):
+            pref = "9"
+        case (True, True):
+            pref = "10"
+    return f"\x1b[{pref}{num}m{s}\x1b[0m"
+
+
+class Colorer:
+    def __init__(self) -> None:
+        self.is_term = sys.stdout.isatty()
+
+    def _colour(self, s: str, num: int, is_bright=False, is_background=False) -> str:
+        if not self.is_term:
+            return s
+        return color(s, num, is_bright, is_background)
+
+    def r(self, s: str) -> str:
+        return self._colour(s, 1)
+
+    def g(self, s: str) -> str:
+        return self._colour(s, 2)
+
+    def y(self, s: str) -> str:
+        return self._colour(s, 3)
+
+    def m(self, s: str) -> str:
+        return self._colour(s, 5)
+
+    def status(self, s: Status) -> str:
+        match s:
+            case "pass":
+                return self.g(s)
+            case "fail":
+                return self.r(s)
+            case "error":
+                return self.m(s)
+            case other:
+                raise ValueError(f"Unknown status '{other}'")
+
 
 @dataclass
 class CommandOutput:
@@ -39,7 +89,7 @@ def from_jso(cls, jso: dict[str, Any]) -> Self:
 @dataclass
 class TestResult:
     test_name: str
-    status: Literal["pass", "fail", "error"]
+    status: Status
     message: str | None
     stderr: str
     return_code: int
@@ -48,14 +98,12 @@ class TestResult:
 
 @dataclass
 class Conformance:
-    strict: bool
     valid: bool
     description: bool | None
 
     @classmethod
     def from_jso(cls, jso: dict[str, Any]) -> Self:
         return cls(
-            strict=jso.get("strict", False),
             valid=jso.get("valid", True),
             description=jso.get("description"),
         )
@@ -80,25 +128,21 @@ def __init__(
         self,
         exclude_patterns: list[re.Pattern] | None = None,
         include_patterns: list[re.Pattern] | None = None,
-        exclude_strict=False,
         exclude_invalid=False,
     ) -> None:
         self.exclude_patterns = exclude_patterns or []
         self.include_patterns = include_patterns or []
 
-        if exclude_strict:
-            self.exclude_patterns.append(re.compile(r"^strict/"))
         if exclude_invalid:
             self.exclude_patterns.append(re.compile(r"^\w+/invalid/"))
 
     def include(self, name: str) -> bool:
         if self.exclude_patterns and any(p.search(name) for p in self.exclude_patterns):
             return False
-        if self.include_patterns and not any(
-            p.search(name) for p in self.include_patterns
-        ):
-            return False
-        return True
+
+        if not self.include_patterns:
+            return True
+        return any(p.search(name) for p in self.include_patterns)
 
 
 def test_path_to_name(fpath: Path, root: Path) -> str:
@@ -110,7 +154,9 @@ def run_test(dingus_cmd: list[str], fpath: Path, test_name: str) -> TestResult:
     test_logger = logger.getChild(test_name)
 
     strictness, validity, *_ = test_name.split("/")
-    if strictness not in ("strict", "spec"):
+    # previously there were "strict" tests which treated schema SHOULDs as MUSTs;
+    # these have since been removed
+    if strictness != "spec":
         raise RuntimeError(f"cannot determine strictness from name: {test_name}")
 
     if validity == "invalid":
@@ -124,6 +170,7 @@ def run_test(dingus_cmd: list[str], fpath: Path, test_name: str) -> TestResult:
         dingus_cmd + [os.fspath(fpath)],
         text=True,
         capture_output=True,
+        check=False,
     )
 
     if res.returncode:
@@ -205,7 +252,7 @@ def main(raw_args=None):
         "--exclude-strict",
         "-S",
         action="store_true",
-        help="exclude strict tests",
+        help="DEPRECATED: exclude strict tests",
     )
     parser.add_argument(
         "--exclude-invalid",
@@ -241,7 +288,12 @@ def main(raw_args=None):
         3: logging.DEBUG,
     }.get(args.verbose, logging.DEBUG)
     logging.basicConfig(level=lvl)
-    logging.debug("Got args: %s", args)
+    logger.debug("Got args: %s", args)
+
+    if args.exclude_strict:
+        logger.warning(
+            "Strict test cases are deprecated; -S/--exclude-strict argument is implicit and will soon be removed."
+        )
 
     if dingus_args is None:
         print(
@@ -267,28 +319,32 @@ def main(raw_args=None):
     req = Requested(
         exclude_patterns=args.exclude_pattern,
         include_patterns=args.include_pattern,
-        exclude_strict=bool(args.exclude_strict),
         exclude_invalid=bool(args.exclude_invalid),
     )
 
     test_paths = ((test_path_to_name(p, dpath), p) for p in dpath.rglob(rglob))
     cases = dict(sorted((n, p) for n, p in test_paths if req.include(n)))
 
+    c = Colorer()
+
     for res in run_all_tests(
         dingus_args,
         cases,
     ):
-        row = [
-            res.test_name,
-            res.status,
-        ]
         if res.status == "pass":
             passes += 1
         elif res.status == "fail":
             failures += 1
         elif res.status == "error":
             errors += 1
 
+        row = [
+            res.test_name,
+            c.status(res.status),
+        ]
+        if res.message:
+            row.append(" ".join(res.message.split()))
+
         print("\t".join(row))
 
     logger.info("Got %s passes, %s failures, %s errors", passes, failures, errors)

examples/label/.config.json

@@ -0,0 +1,3 @@
+{
+    "schema": "schemas/label.schema"
+}

examples/multiscales/.config.json

@@ -0,0 +1,3 @@
+{
+  "schema": "schemas/image.schema"
+}

examples/plate/.config.json

@@ -0,0 +1,3 @@
+{
+  "schema": "schemas/plate.schema"
+}

examples/well/.config.json

@@ -0,0 +1,3 @@
+{
+  "schema": "schemas/well.schema"
+}

index.md

@@ -429,7 +429,7 @@ The following transformations are supported:
 | [`bijection`](#bijection-md) | `"forward":Transformation`<br>`"inverse":Transformation` | An invertible transformation providing an explicit forward transformation and its inverse. |
 | [`byDimension`](#bydimension-md) | `"transformations":List[Transformation]`.<br>Transformations in the array MUST have<br>`"inputAxes": List[number]`, <br> and `"outputAxes": List[number]` | A high dimensional transformation using lower dimensional transformations on subsets of dimensions. |
 
-The parameter values (e.g., `scale` for a [scale transformation](#scale-md)) MUST be compatible with input and output space dimensionality (see details). 
+The parameter values (e.g., `scale` for a [scale transformation](#scale-md)) MUST be compatible with input and output space dimensionality (see details).
 
 The `input` and `output` fields are objects structured as follows:
 
@@ -471,7 +471,7 @@ Depending on which, different constraints apply to the transformations, as descr
   - Both `input` and `output` MUST specify a coordinate system `name`.
   - `path` is required when referencing a coordinate system in a multiscale image subgroup;
     it MAY be omitted or null when referencing a coordinate system defined in the scene's own `coordinateSystems`.
-  
+
 
 In any context, the values given for `name` and `path` provide an unambiguous reference to a named coordinate system.
 If the `path` field is null or omitted, this is to be interpreted as referring to a named coordinate system in the same `zarr.json` file.
@@ -1107,7 +1107,7 @@ An exact reproducibility of pixel values for images transformed and resampled by
 
 The multiscale group at `path` MUST satisfy:
   - **Dimensionality**: If the input coordinate system has `N` axes, the multiscale image at location `path` MUST have `N+1` dimensions.
-  - **Vector dimension length**: 
+  - **Vector dimension length**:
     - For `coordinates` transformations, the length of the array along the `coordinate` dimension (last axis) MUST equal `M`,
       the number of axes in the output coordinate system.
     - For `displacements` transformations, the length of the array along the `displacement` dimension (last axis) MUST equal `N`,
@@ -1407,7 +1407,7 @@ In this example, a multiscales group containing labels is located at `labels/lab
 :::{dropdown} Example: Complete multiscales metadata
 
 A complete example of json-file for a 5D (TCZYX) multiscales with 3 resolution levels could look like this:
-```{literalinclude} examples/multiscales_strict/multiscales_example.json
+```{literalinclude} examples/multiscales/multiscales_example.json
 :language: json
 ```
 :::
@@ -1562,7 +1562,7 @@ In the `zarr.json` under the image.zarr group, an explicit `identity` transform
 the coordinate system named `"physical"` in the multiscales metadata of the original image is the same as
 the coordinate system named `"physical"` in the multiscales metadata of the label image:
 
-```{literalinclude} examples/multiscales_strict/multiscale_reference_to_label.json
+```{literalinclude} examples/multiscales/multiscale_reference_to_label.json
 :language: json
 ```
 
@@ -1586,7 +1586,7 @@ a coordinate system named `"physical"` serves as the "[intrinsic](#spec:hint:mul
 The `image-label` field contains information about the source image and display colors for the label image,
 i.e., a label image in which 0s and 1s represent intercellular and cellular space, respectively:
 
-```{literalinclude} examples/label_strict/colors_properties.json
+```{literalinclude} examples/label/colors_properties.json
 :language: json
 ```
 
@@ -1665,14 +1665,14 @@ The `rowIndex`, `columnIndex`, and `path` MUST all refer to the same row/column
 For example the following JSON object defines a plate with two acquisitions and 6 wells (2 rows and 3 columns),
 containing up to 2 fields of view per acquisition.
 
-```{literalinclude} examples/plate_strict/plate_6wells.json
+```{literalinclude} examples/plate/plate_6wells.json
 :language: json
 ```
 
 The following JSON object defines a sparse plate with one acquisition and 2 wells in a 96 well plate,
 containing one field of view per acquisition.
 
-```{literalinclude} examples/plate_strict/plate_2wells.json
+```{literalinclude} examples/plate/plate_2wells.json
 :language: json
 ```
 :::
@@ -1701,14 +1701,14 @@ For example the following JSON object defines a well with four fields of view.
 The first two fields of view were part of the first acquisition
 while the last two fields of view were part of the second acquisition.
 
-```{literalinclude} examples/well_strict/well_4fields.json
+```{literalinclude} examples/well/well_4fields.json
 :language: json
 ```
 
 The following JSON object defines a well with two fields of view in a plate with four acquisitions.
 The first field is part of the first acquisition, and the second field is part of the last acquisition.
 
-```{literalinclude} examples/well_strict/well_2fields.json
+```{literalinclude} examples/well/well_2fields.json
 :language: json
 ```
 :::
@@ -1866,7 +1866,7 @@ If they do so, it is RECOMMENDED that the scene's first entry under the `coordin
 If no coordinate system is defined therein, but only in the respective linked multiscale groups,
 viewers may want to expose a choice for the user to select a coordinate system for display when opening the dataset for the first time.
 
- 
+
 
 ```
 

pre_build.py

@@ -87,14 +87,11 @@ def build_json_schemas():
 """
 
     for schema_file in schema_files:
-        if 'strict' in schema_file:
-            continue  # skip strict schemas
-
         print(f'Processing {schema_file}...')
         output_path_md = os.path.join(output_directory, f"{Path(schema_file).stem}" + ".md")
         output_path_html = os.path.join(output_directory, f"{Path(schema_file).stem}" + ".html")
         os.makedirs(os.path.dirname(output_path_md), exist_ok=True)
-        os.makedirs(os.path.dirname(output_path_html), exist_ok=True)        
+        os.makedirs(os.path.dirname(output_path_html), exist_ok=True)
 
         # Generate the documentation
         try: