From 1f0e8148c7e6a00636e1d1139a6748a0a8ee79ff Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 12 Oct 2026 17:04:23 +0200 Subject: [PATCH 1/8] Render structured JSON results as single comparison cards --- AGENTS.md | 2 +- README.md | 7 + src/vlm_exam/visualization/structured.py | 360 +++++++++++++++++++++++ src/vlm_exam/visualization/text.py | 178 ++--------- tests/test_cli_run.py | 162 ++++++++-- 5 files changed, 534 insertions(+), 175 deletions(-) create mode 100644 src/vlm_exam/visualization/structured.py diff --git a/AGENTS.md b/AGENTS.md index 1603813..9dd40be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -450,7 +450,7 @@ renderer and PNG fingerprints in the compact chart manifest. Before updating this PR, run Ruff check/format, existing tests, `validate`, and `summary --check` (including text coverage and chart freshness). Verify cards -for strings, JSON, localization, empty/invalid responses and pagination when +for strings, JSON, localization, empty/invalid responses and bounded single-card JSON when changing rendering. Add focused regressions to existing test modules; do not create new test files. Moving unchanged result files does not require rebuilding PNGs or recomputing inference. diff --git a/README.md b/README.md index 8ef540e..f5e19ea 100644 --- a/README.md +++ b/README.md @@ -463,3 +463,10 @@ explicitly unscored, with coverage reported for each metric. See [AGENTS.md](AGENTS.md#mixed-text-benchmark) for dataset import, inference, coordinate-format requirements, validation and rendering commands. Charts use the shared [leaderboard directory](visualizations/leaderboards/). + +Structured JSON cards show one image per sample, including large responses. +Small objects show all fields; larger responses add a field agreement map grouped +by JSON parent paths and prioritize mismatches in the value tiles. Purple marks +differences, with the expected value struck through above the model value. +FIELD F1 uses the shared detection-score styling. Long values may be shortened +with an ellipsis; full responses remain in the source result file. diff --git a/src/vlm_exam/visualization/structured.py b/src/vlm_exam/visualization/structured.py new file mode 100644 index 0000000..3a666d2 --- /dev/null +++ b/src/vlm_exam/visualization/structured.py @@ -0,0 +1,360 @@ +# Copyright 2026 Roboflow, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from functools import lru_cache +from itertools import groupby +from typing import Any + +import matplotlib.pyplot as plt +from matplotlib import font_manager +from matplotlib.ft2font import FT2Font +from PIL import Image, ImageDraw, ImageFont + +from vlm_exam.tasks.text_scoring import equal, flatten, parse_json +from vlm_exam.visualization import theme + +_WIDTH = 879 +_HEIGHT = 604 +_MISSING = object() +_MAP_MATCH = "#E4E3E9" +_MAP_CHANGED = "#8253BC" +_CARD_MATCH = "#F3F2F5" +_CARD_CHANGED = "#F0EAF8" +_VALUE_CHANGED = "#743AC5" + + +@dataclass(frozen=True) +class _Field: + path: tuple[str | int, ...] + expected: Any + predicted: Any + + @property + def matches(self) -> bool: + return ( + self.expected is not _MISSING + and self.predicted is not _MISSING + and equal(self.expected, self.predicted) + ) + + +@dataclass +class _Tile: + field: _Field + x: int + y: int + width: int + height: int + labels: list[str] + values: list[list[str]] + size: int + + +@lru_cache(maxsize=256) +def _coverage(path: str) -> frozenset[int]: + try: + return frozenset(FT2Font(path).get_charmap()) + except (OSError, RuntimeError): + return frozenset() + + +@lru_cache(maxsize=1) +def _fallbacks() -> tuple[str, ...]: + fonts = sorted( + font_manager.fontManager.ttflist, + key=lambda font: ( + not any( + name in font.name.lower() for name in ("unicode", "noto", "hiragino") + ), + font.style != "normal", + abs( + ( + font_manager.weight_dict.get(font.weight, 400) + if isinstance(font.weight, str) + else font.weight + ) + - 400 + ), + font.name, + font.fname, + ), + ) + return tuple(dict.fromkeys(font.fname for font in fonts)) + + +@lru_cache(maxsize=256) +def _font(path: str, size: int) -> ImageFont.FreeTypeFont: + return ImageFont.truetype(path, size) + + +def _text_font(value: str, size: int) -> ImageFont.FreeTypeFont: + primary = font_manager.findfont(theme.load_fonts().regular) + points = {ord(character) for character in value} + if points <= _coverage(primary): + return _font(primary, size) + for path in _fallbacks(): + if points <= _coverage(path): + return _font(path, size) + return _font(primary, size) + + +def _path(path: tuple[str | int, ...]) -> str: + label = "" + for part in path: + if isinstance(part, int): + label += f"[{part}]" + elif part.isidentifier(): + label += ("." if label else "") + part + else: + label += "[" + json.dumps(part, ensure_ascii=False) + "]" + return label or "$" + + +def _value(value: Any) -> str: + if value is _MISSING: + return "[missing]" + if not isinstance(value, str): + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + try: + json.loads(value) + ambiguous = True + except ValueError: + ambiguous = False + if not value or ambiguous or value == "[missing]" or value != value.strip(): + return json.dumps(value, ensure_ascii=False) + return value.replace("\n", " ↵ ").replace("\r", " ↵ ").replace("\t", " ⇥ ") + + +def _fields(expected: Any, predicted: Any) -> list[_Field]: + reference, model = flatten(expected), flatten(predicted) + paths = list(reference) + [path for path in model if path not in reference] + return [ + _Field(path, reference.get(path, _MISSING), model.get(path, _MISSING)) + for path in paths + ] + + +def _wrap(value: str, width: int, size: int, limit: int = 30) -> list[str]: + font = _text_font(value, size) + lines = [] + while value and len(lines) < limit: + low, high = 1, min(len(value), 2048) + while low < high: + middle = (low + high + 1) // 2 + if font.getlength(value[:middle]) <= width: + low = middle + else: + high = middle - 1 + lines.append(value[:low]) + value = value[low:] + if value: + while lines[-1] and font.getlength(lines[-1] + "…") > width: + lines[-1] = lines[-1][:-1] + lines[-1] += "…" + return lines or [""] + + +def _layout( + fields: list[_Field], columns: int, size: int, top: int = 5, limit: int = 30 +) -> tuple[list[_Tile], int]: + width = (_WIDTH - (columns - 1) * 18) // columns + bottoms = [top] * columns + tiles = [] + for field in fields: + labels = _wrap(_path(field.path), width - 28, 14, 2) + values = ( + [] + if field.matches + else [_wrap(_value(field.expected), width - 28, size, limit)] + ) + values.append(_wrap(_value(field.predicted), width - 28, size, limit)) + height = ( + len(labels) * 18 + 38 + sum(len(lines) * (size + 5) for lines in values) + ) + column = min(range(columns), key=bottoms.__getitem__) + tiles.append( + _Tile( + field, + column * (width + 18), + bottoms[column], + width, + height, + labels, + values, + size, + ) + ) + bottoms[column] += height + 9 + return tiles, max(bottoms) + + +def _map_layout(fields: list[_Field], height: int) -> list[tuple[int, int, int, bool]]: + groups = [ + list(group) for _, group in groupby(fields, key=lambda field: field.path[:-1]) + ] + for size in range(22, 0, -1): + x, y = 0, 4 + cells = [] + for group in groups: + width = len(group) * (size + 2) - 2 + if x and x + width > _WIDTH: + x, y = 0, y + size + 9 + for field in group: + if x + size > _WIDTH: + x, y = 0, y + size + 9 + cells.append((x, y, size, field.matches)) + x += size + 2 + x += 8 + if y + size + 4 <= height: + return cells + # Above pixel capacity, a bin is changed if any of its fields differs. + capacity = (_WIDTH // 3) * ((height - 8) // 3) + step = math.ceil(len(fields) / capacity) + return [ + ( + (index % (_WIDTH // 3)) * 3, + 4 + (index // (_WIDTH // 3)) * 3, + 2, + all(field.matches for field in fields[offset : offset + step]), + ) + for index, offset in enumerate(range(0, len(fields), step)) + ] + + +def _draw_tiles(draw: ImageDraw.ImageDraw, tiles: list[_Tile]) -> None: + for tile in tiles: + x, y = tile.x, tile.y + draw.rounded_rectangle( + (x, y, x + tile.width - 1, y + tile.height - 1), + radius=7, + fill=_CARD_MATCH if tile.field.matches else _CARD_CHANGED, + ) + x, y = x + 14, y + 11 + for line in tile.labels: + draw.text( + (x, y), line, fill=theme.TEXT_SECONDARY, font=_text_font(line, 14) + ) + y += 18 + y += 7 + for index, lines in enumerate(tile.values): + strike = not tile.field.matches and index == 0 + color = ( + theme.TEXT_SECONDARY + if strike + else theme.TEXT_PRIMARY + if tile.field.matches + else _VALUE_CHANGED + ) + for line in lines: + font = _text_font(line, tile.size) + draw.text((x, y), line, fill=color, font=font) + if strike: + bounds = draw.textbbox((x, y), line, font=font) + middle = (bounds[1] + bounds[3]) / 2 + draw.line( + (x, middle, x + font.getlength(line), middle), + fill=color, + width=1, + ) + y += tile.size + 5 + + +def _panel(expected: Any, response: str) -> Image.Image: + panel = Image.new("RGB", (_WIDTH, _HEIGHT), theme.BACKGROUND_COLOR) + draw = ImageDraw.Draw(panel) + try: + predicted, _ = parse_json(response) + except (ValueError, TypeError, RecursionError): + message = "Invalid JSON" if response.strip() else "Empty response" + draw.text((14, 14), message, fill=_VALUE_CHANGED, font=_text_font(message, 21)) + preview = _wrap( + response.replace("\n", " ↵ ").replace("\r", " ↵ ").replace("\t", " ⇥ ") + if response + else "(empty response)", + _WIDTH - 28, + 20, + 4, + ) + for index, line in enumerate(preview): + draw.text( + (14, 57 + index * 25), + line, + fill=theme.TEXT_PRIMARY, + font=_text_font(line, 20), + ) + draw.line((0, 175, _WIDTH - 1, 175), fill=theme.DIVIDER_COLOR, width=1) + draw.text( + (14, 192), + "Expected", + fill=theme.TEXT_SECONDARY, + font=_text_font("Expected", 14), + ) + reference = _fields(expected, expected) + for count in range(min(6, len(reference)), 0, -1): + tiles, bottom = _layout(reference[:count], 2, 20, top=225, limit=2) + if bottom <= _HEIGHT: + _draw_tiles(draw, tiles) + break + return panel + fields = _fields(expected, predicted) + for columns, size in ((2, 21), (2, 20), (2, 19), (3, 19), (3, 18), (3, 17)): + tiles, bottom = _layout(fields, columns, size) + if bottom <= _HEIGHT: + _draw_tiles(draw, tiles) + return panel + if len(fields) <= 2: + tiles, _ = _layout(fields, 1, 20, limit=9 if len(fields) == 1 else 3) + _draw_tiles(draw, tiles) + return panel + columns = 40 if len(fields) > 160 else 24 if len(fields) > 72 else 16 + rows = math.ceil(len(fields) / columns) + bottom = int(rows * min(16, 208 / rows) + 4) + cells = _map_layout(fields, bottom + 10) + for x, y, size, matches in cells: + draw.rectangle( + (x, y, x + size - 1, y + size - 1), + fill=_MAP_MATCH if matches else _MAP_CHANGED, + ) + draw.line((0, bottom + 17, _WIDTH - 1, bottom + 17), fill="#E2E2E9", width=1) + ranked = sorted(fields, key=lambda field: field.matches) + for count in range(min(12, len(fields) - 1), 0, -1): + tiles, height = _layout(ranked[:count], 2, 20, bottom + 35, 2) + if height <= _HEIGHT: + _draw_tiles(draw, tiles) + break + return panel + + +def draw_structured_panel(rail: plt.Axes, expected: Any, response: str) -> None: + """Draw a bounded field comparison with a grouped map for large JSON. + + Args: + rail: Native hero-card right-hand axes. + expected: Parsed reference JSON. + response: Model response, including potentially invalid JSON. + """ + rail.imshow( + _panel(expected, response), + extent=(0, 1, 0.279, 0.869), + aspect="auto", + interpolation="nearest", + zorder=0, + ) + rail.set_xlim(0, 1) + rail.set_ylim(0, 1) diff --git a/src/vlm_exam/visualization/text.py b/src/vlm_exam/visualization/text.py index ffed359..5ae3a85 100644 --- a/src/vlm_exam/visualization/text.py +++ b/src/vlm_exam/visualization/text.py @@ -14,10 +14,7 @@ from __future__ import annotations -import difflib -import json import math -import re import textwrap from typing import Any @@ -32,12 +29,11 @@ from vlm_exam.tasks.text_scoring import match_regions, normalize, parse_json from vlm_exam.visualization import theme from vlm_exam.visualization.cases import ( - draw_diff_line, - draw_verdict, plot_qa_card, plot_transcription_card, ) from vlm_exam.visualization.detection import region_diff_image +from vlm_exam.visualization.structured import draw_structured_panel def _frame( @@ -52,78 +48,6 @@ def _frame( return figure, rail -def _token_diff(left: str, right: str) -> tuple[list[tuple[str, str | None]], ...]: - pattern = ( - r'"(?:[^"\\]|\\.)*"|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|true|false|null|[^\s]' - ) - tokens = [list(re.finditer(pattern, value)) for value in (left, right)] - changes: list[set[int]] = [set(), set()] - matcher = difflib.SequenceMatcher( - None, - [t.group() for t in tokens[0]], - [t.group() for t in tokens[1]], - autojunk=False, - ) - for operation, a, b, c, d in matcher.get_opcodes(): - if operation != "equal": - changes[0].update(range(a, b)) - changes[1].update(range(c, d)) - output = [] - for value, parts, changed, kind in zip( - (left, right), tokens, changes, ("expected", "predicted") - ): - runs: list[tuple[str, str | None]] = [] - position = 0 - for index, token in enumerate(parts): - runs.append((value[position : token.start()], None)) - runs.append((token.group(), kind if index in changed else None)) - position = token.end() - runs.append((value[position:], None)) - output.append(runs) - return tuple(output) - - -def _json_diff( - expected: Any, predicted: Any -) -> list[tuple[str, list[tuple[str, str | None]]]]: - left, right = ( - json.dumps(value, indent=2, ensure_ascii=False, sort_keys=True).splitlines() - for value in (expected, predicted) - ) - output = [] - for operation, a, b, c, d in difflib.SequenceMatcher( - None, left, right, autojunk=False - ).get_opcodes(): - if operation == "equal": - output.extend((" ", [(line, None)]) for line in left[a:b]) - elif operation == "replace" and b - a == d - c: - for expected_line, predicted_line in zip(left[a:b], right[c:d]): - expected_runs, predicted_runs = _token_diff( - expected_line, predicted_line - ) - output.extend([("−", expected_runs), ("+", predicted_runs)]) - else: - output.extend(("−", [(line, "expected")]) for line in left[a:b]) - output.extend(("+", [(line, "predicted")]) for line in right[c:d]) - return output - - -def _wrap_runs( - runs: list[tuple[str, str | None]], width: int -) -> list[list[tuple[str, str | None]]]: - characters = [(character, kind) for text, kind in runs for character in text] - output = [] - for offset in range(0, len(characters), width): - line: list[tuple[str, str | None]] = [] - for character, kind in characters[offset : offset + width]: - if line and line[-1][1] == kind: - line[-1] = (line[-1][0] + character, kind) - else: - line.append((character, kind)) - output.append(line) - return output or [[]] - - def _json_cards( image: Image.Image, sample: TextSample, @@ -131,74 +55,34 @@ def _json_cards( model: str, config: VisualizationConfig, ) -> list[plt.Figure]: - invalid = False - try: - predicted, _ = parse_json(result.predicted) - lines = _json_diff(sample.answer, predicted) - except (ValueError, TypeError, RecursionError): - invalid = True - reference = json.dumps( - sample.answer, indent=2, ensure_ascii=False, sort_keys=True - ) - lines = [("−", [(line, "expected")]) for line in reference.splitlines()] - lines += [(" ", [("MODEL RESPONSE — INVALID JSON", None)])] - lines += [ - ("+", [(line, "predicted")]) - for line in (result.predicted.splitlines() or ["(empty response)"]) - ] - width_inches = theme.HERO_RAIL_RECT[2] * theme.CARD_FIGURE_SIZE[0] - height_inches = theme.HERO_RAIL_RECT[3] * theme.CARD_FIGURE_SIZE[1] - for font_size in (14, 13, 12, 11): - width = int(width_inches * 72 / font_size / theme.MONO_ADVANCE_EM) - 3 - wrapped = [ - (prefix if index == 0 else " ", row) - for prefix, runs in lines - for index, row in enumerate(_wrap_runs(runs, width)) - ] - step = font_size * 1.42 / 72 / height_inches - capacity = int((0.725 - 0.115) / step) - if len(wrapped) <= capacity: - break - pages = max(1, math.ceil(len(wrapped) / capacity)) - figures = [] - for page in range(pages): - figure, rail = _frame(image, model, config, "DATA EXTRACTION") - draw_verdict(rail, result.correct) - rail.text( - 0, - 0.792, - "JSON RESPONSE" + (" · INVALID JSON" if invalid else ""), - font=theme.load_fonts().bold, - fontsize=9.5, - color=theme.PANEL_LABEL_COLOR, - ) - edge = theme.draw_legend_chip( - rail, 1, 0.797, "+ model", theme.FAILURE_TEXT_COLOR - ) - theme.draw_legend_chip( - rail, edge, 0.797, "− expected", theme.SUCCESS_TEXT_COLOR - ) - rail.plot([0, 1], [0.768, 0.768], color=theme.DIVIDER_COLOR, linewidth=1) - for index, (prefix, runs) in enumerate( - wrapped[page * capacity : (page + 1) * capacity] - ): - y = 0.725 - index * step - color = ( - theme.SUCCESS_TEXT_COLOR - if prefix == "−" - else theme.FAILURE_TEXT_COLOR - if prefix == "+" - else theme.TEXT_PRIMARY - ) - draw_diff_line(rail, 0, y, [(prefix, None)], font_size, color) - draw_diff_line(rail, 0.036, y, runs, font_size) - theme.draw_brand_footer( - rail, - "evaluated via exact field values" - + (f" · {page + 1}/{pages}" if pages > 1 else ""), - ) - figures.append(figure) - return figures + figure, rail = _frame(image, model, config, "DATA EXTRACTION") + draw_structured_panel(rail, sample.answer, result.predicted) + fonts = theme.load_fonts() + rail.plot([0, 1], [0.262, 0.262], color=theme.DIVIDER_COLOR, lw=1, clip_on=False) + rail.text( + 0.5, + 0.212, + "FIELD F1", + fontsize=9.5, + ha="center", + va="center", + color=theme.PANEL_LABEL_COLOR, + font=fonts.bold, + ) + score = result.metadata.get("score") + scored = type(score) in (int, float) and math.isfinite(score) and 0 <= score <= 1 + rail.text( + 0.5, + 0.142, + f"{score * 100:.1f}%" if scored else "—", + fontsize=26, + ha="center", + va="center", + font=fonts.display, + color=theme.score_color(score) if scored else theme.TEXT_SECONDARY, + ) + theme.draw_brand_footer(rail, "evaluated via exact field values") + return [figure] def _region_rows( @@ -443,7 +327,7 @@ def plot_text_cards( config: VisualizationConfig, uploaded_size: tuple[int, int] | None = None, ) -> list[plt.Figure]: - """Render native text cards, with additional pages for long JSON responses. + """Render one native hero card for each text sample. Args: image: Original source image. @@ -455,7 +339,7 @@ def plot_text_cards( uploaded_size: Recorded provider image dimensions for pixel conversion. Returns: - One or more native hero-card figures. + A single native hero-card figure in a list. """ if sample.category == "structured": return _json_cards(image, sample, result, model, config) diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index ad828df..245457b 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -14,6 +14,7 @@ from __future__ import annotations +import json from dataclasses import dataclass from pathlib import Path from typing import Any @@ -237,52 +238,159 @@ def test_resume_replaces_failures_and_deletes_source( assert "Removed resumed file" in result.output -@pytest.mark.parametrize("prediction", ["", '{"total":']) +@pytest.fixture() +def card_fonts(monkeypatch: pytest.MonkeyPatch) -> None: + from matplotlib.font_manager import FontProperties + from PIL import Image + + from vlm_exam.visualization import theme + + monkeypatch.setattr( + theme, "fetch_logo", lambda *args, **kwargs: Image.new("RGBA", (30, 30)) + ) + font = FontProperties(family="DejaVu Sans") + fonts = theme.FontSet(font, font, font, font, font) + monkeypatch.setattr(theme, "load_fonts", lambda: fonts) + + +@pytest.mark.parametrize("prediction", ["", '{"total":', '{"x":NaN}', '{"x":1,"x":2}']) def test_invalid_json_card_keeps_reference( - prediction: str, monkeypatch: pytest.MonkeyPatch + prediction: str, monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from PIL import ImageDraw + + from vlm_exam.visualization import structured + + drawn: list[str] = [] + original = ImageDraw.ImageDraw.text + + def capture(self: Any, position: Any, text: str, *args: Any, **kwargs: Any) -> Any: + drawn.append(text) + return original(self, position, text, *args, **kwargs) + + monkeypatch.setattr(ImageDraw.ImageDraw, "text", capture) + panel = structured._panel({"total": "42.00"}, prediction) + assert panel.size == (879, 604) + assert "total" in drawn and '"42.00"' in drawn + assert "Invalid JSON" in drawn if prediction else "Empty response" in drawn + assert prediction in drawn if prediction else "(empty response)" in drawn + + +@pytest.mark.parametrize("score", [0.0, 0.8, 0.95, None]) +def test_large_json_is_one_card_with_native_field_score( + score: float | None, card_fonts: None ) -> None: from types import SimpleNamespace import matplotlib.pyplot as plt - from matplotlib.font_manager import FontProperties from PIL import Image + from vlm_exam.config import DisplayConfig, ModelDisplay from vlm_exam.visualization import text as cards - font = FontProperties() - monkeypatch.setattr(cards.theme, "load_fonts", lambda: SimpleNamespace(bold=font)) - monkeypatch.setattr(cards, "draw_verdict", lambda *args: None) - monkeypatch.setattr(cards.theme, "draw_legend_chip", lambda *args: 0.5) - monkeypatch.setattr(cards.theme, "draw_brand_footer", lambda *args: None) - - def frame(*args: Any) -> tuple[Any, Any]: - figure = plt.figure() - return figure, figure.add_axes([0, 0, 1, 1]) - - monkeypatch.setattr(cards, "_frame", frame) - drawn = [] - monkeypatch.setattr( - cards, - "draw_diff_line", - lambda rail, x, y, spans, *args: drawn.extend(spans), + answer = {"rows": [[str(i + j) for j in range(10)] for i in range(62)]} + config = DisplayConfig( + labs={"lab": LabConfig("Lab", "#000000", "")}, + models={"test": ModelDisplay("Test", "lab")}, ) + result = SimpleNamespace(predicted=json.dumps(answer), metadata={"score": score}) figures = cards._json_cards( Image.new("RGB", (20, 20)), - SimpleNamespace(answer={"total": "42.00"}), - SimpleNamespace(predicted=prediction, correct=False), - "model", - None, + SimpleNamespace(answer=answer), + result, + "test", + config, ) try: - text = "".join(span[0] for span in drawn) - assert '"total": "42.00"' in text - assert "INVALID JSON" in text - assert prediction in text if prediction else "(empty response)" in text + assert len(figures) == 1 + rail = figures[0].axes[1] + texts = {text.get_text(): text for text in rail.texts} + value = f"{score * 100:.1f}%" if score is not None else "—" + assert texts["FIELD F1"].get_position() == (0.5, 0.212) + assert texts[value].get_position() == (0.5, 0.142) + assert texts[value].get_color() == ( + cards.theme.score_color(score) + if score is not None + else cards.theme.TEXT_SECONDARY + ) + assert len(rail.images) == 1 finally: for figure in figures: plt.close(figure) +def test_json_fields_preserve_types_and_missing_extra_paths() -> None: + from vlm_exam.visualization.structured import _fields, _path, _value + + fields = _fields( + {"zero": 0, "null": None, "empty": [], "missing": "x"}, + {"zero": False, "null": "null", "empty": {}, "extra": True}, + ) + assert not any(field.matches for field in fields) + assert [field.path for field in fields][-1] == ("extra",) + assert _value(fields[3].predicted) == "[missing]" + assert _value("null") != _value(None) + assert _value("1.5") != _value(1.5) + assert _path(("a.b",)) != _path(("a", "b")) + + +def test_json_map_is_conditional_and_prioritizes_differences( + monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from vlm_exam.visualization import structured + + drawn: list[Any] = [] + maps: list[Any] = [] + draw_tiles = structured._draw_tiles + map_layout = structured._map_layout + + def tiles(draw: Any, items: list[Any]) -> None: + drawn.extend(items) + assert all(tile.y + tile.height <= 604 for tile in items) + draw_tiles(draw, items) + + def cells(fields: list[Any], height: int) -> list[Any]: + result = map_layout(fields, height) + maps.extend(result) + return result + + monkeypatch.setattr(structured, "_draw_tiles", tiles) + monkeypatch.setattr(structured, "_map_layout", cells) + structured._panel({"a": None}, '{"a":null}') + assert not maps + drawn.clear() + expected = {"rows": [["ok"] * 10 for _ in range(62)]} + predicted = json.loads(json.dumps(expected)) + predicted["rows"][-1][-1] = "wrong" + structured._panel(expected, json.dumps(predicted)) + assert len(maps) == 620 + assert sum(not match for _, _, _, match in maps) == 1 + assert drawn[0].field.path == ("rows", 61, 9) + assert len(drawn) < 620 + assert all( + 0 <= x < x + size <= 879 and 0 <= y < y + size <= 222 for x, y, size, _ in maps + ) + + +@pytest.mark.parametrize("fields", [1, 2, 4]) +def test_long_json_strings_remain_within_card( + fields: int, monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from vlm_exam.visualization import structured + + original = structured._draw_tiles + + def check(draw: Any, tiles: list[Any]) -> None: + assert tiles + assert all(tile.y + tile.height <= 604 for tile in tiles) + original(draw, tiles) + + monkeypatch.setattr(structured, "_draw_tiles", check) + expected = {str(i): "abc " * 2000 for i in range(fields)} + predicted = {str(i): "xyz " * 2000 for i in range(fields)} + assert structured._panel(expected, json.dumps(predicted)).size == (879, 604) + + @pytest.mark.parametrize("count", [1, 10]) def test_leaderboard_export_has_fixed_visible_padding( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, count: int From 5241365221c5fe515f6f181794cb0b2d7e1223b5 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 12 Oct 2026 17:17:28 +0200 Subject: [PATCH 2/8] Handle mixed-script fonts and deeply nested JSON in cards --- src/vlm_exam/visualization/structured.py | 60 +++++++++++++----- tests/test_cli_run.py | 80 ++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 15 deletions(-) diff --git a/src/vlm_exam/visualization/structured.py b/src/vlm_exam/visualization/structured.py index 3a666d2..117a2c2 100644 --- a/src/vlm_exam/visualization/structured.py +++ b/src/vlm_exam/visualization/structured.py @@ -26,7 +26,7 @@ from matplotlib.ft2font import FT2Font from PIL import Image, ImageDraw, ImageFont -from vlm_exam.tasks.text_scoring import equal, flatten, parse_json +from vlm_exam.tasks.text_scoring import equal, parse_json from vlm_exam.visualization import theme _WIDTH = 879 @@ -77,7 +77,11 @@ def _coverage(path: str) -> frozenset[int]: @lru_cache(maxsize=1) def _fallbacks() -> tuple[str, ...]: fonts = sorted( - font_manager.fontManager.ttflist, + ( + font + for font in font_manager.fontManager.ttflist + if "lastresort" not in font.name.lower().replace(" ", "") + ), key=lambda font: ( not any( name in font.name.lower() for name in ("unicode", "noto", "hiragino") @@ -108,10 +112,16 @@ def _text_font(value: str, size: int) -> ImageFont.FreeTypeFont: points = {ord(character) for character in value} if points <= _coverage(primary): return _font(primary, size) + best = primary + supported = len(points & _coverage(primary)) for path in _fallbacks(): - if points <= _coverage(path): + coverage = _coverage(path) + if points <= coverage: return _font(path, size) - return _font(primary, size) + count = len(points & coverage) + if count > supported: + best, supported = path, count + return _font(best, size) def _path(path: tuple[str | int, ...]) -> str: @@ -134,6 +144,8 @@ def _value(value: Any) -> str: try: json.loads(value) ambiguous = True + except RecursionError: + ambiguous = True except ValueError: ambiguous = False if not value or ambiguous or value == "[missing]" or value != value.strip(): @@ -141,8 +153,27 @@ def _value(value: Any) -> str: return value.replace("\n", " ↵ ").replace("\r", " ↵ ").replace("\t", " ⇥ ") +def _leaves(value: Any) -> dict[tuple[str | int, ...], Any]: + leaves = {} + pending: list[tuple[tuple[str | int, ...], Any]] = [((), value)] + while pending: + path, current = pending.pop() + if isinstance(current, dict) and current: + pending.extend( + ((*path, key), child) for key, child in reversed(current.items()) + ) + elif isinstance(current, list) and current: + pending.extend( + ((*path, index), current[index]) + for index in range(len(current) - 1, -1, -1) + ) + else: + leaves[path] = current + return leaves + + def _fields(expected: Any, predicted: Any) -> list[_Field]: - reference, model = flatten(expected), flatten(predicted) + reference, model = _leaves(expected), _leaves(predicted) paths = list(reference) + [path for path in model if path not in reference] return [ _Field(path, reference.get(path, _MISSING), model.get(path, _MISSING)) @@ -246,14 +277,15 @@ def _draw_tiles(draw: ImageDraw.ImageDraw, tiles: list[_Tile]) -> None: fill=_CARD_MATCH if tile.field.matches else _CARD_CHANGED, ) x, y = x + 14, y + 11 + label_font = _text_font(_path(tile.field.path), 14) for line in tile.labels: - draw.text( - (x, y), line, fill=theme.TEXT_SECONDARY, font=_text_font(line, 14) - ) + draw.text((x, y), line, fill=theme.TEXT_SECONDARY, font=label_font) y += 18 y += 7 for index, lines in enumerate(tile.values): strike = not tile.field.matches and index == 0 + value = tile.field.expected if strike else tile.field.predicted + font = _text_font(_value(value), tile.size) color = ( theme.TEXT_SECONDARY if strike @@ -262,7 +294,6 @@ def _draw_tiles(draw: ImageDraw.ImageDraw, tiles: list[_Tile]) -> None: else _VALUE_CHANGED ) for line in lines: - font = _text_font(line, tile.size) draw.text((x, y), line, fill=color, font=font) if strike: bounds = draw.textbbox((x, y), line, font=font) @@ -283,20 +314,19 @@ def _panel(expected: Any, response: str) -> Image.Image: except (ValueError, TypeError, RecursionError): message = "Invalid JSON" if response.strip() else "Empty response" draw.text((14, 14), message, fill=_VALUE_CHANGED, font=_text_font(message, 21)) - preview = _wrap( + raw = ( response.replace("\n", " ↵ ").replace("\r", " ↵ ").replace("\t", " ⇥ ") if response - else "(empty response)", - _WIDTH - 28, - 20, - 4, + else "(empty response)" ) + preview = _wrap(raw, _WIDTH - 28, 20, 4) + preview_font = _text_font(raw, 20) for index, line in enumerate(preview): draw.text( (14, 57 + index * 25), line, fill=theme.TEXT_PRIMARY, - font=_text_font(line, 20), + font=preview_font, ) draw.line((0, 175, _WIDTH - 1, 175), fill=theme.DIVIDER_COLOR, width=1) draw.text( diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index 245457b..e63b34e 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -451,3 +451,83 @@ def test_text_run_uses_shared_output_default( ) assert result.exit_code == 0, result.output assert len(list((tmp_path / "results").glob("text_*.jsonl"))) == 1 + + +def test_json_wrapping_keeps_the_measured_font( + monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from matplotlib import font_manager + from PIL import ImageDraw, ImageFont + + from vlm_exam.visualization import structured + + primary = font_manager.findfont("DejaVu Sans Mono") + fallback = font_manager.findfont("DejaVu Sans") + + def choose(value: str, size: int) -> Any: + return ImageFont.truetype(fallback if "日本語" in value else primary, size) + + original = ImageDraw.ImageDraw.text + widths: list[float] = [] + + def capture(self: Any, position: Any, text: str, *args: Any, **kwargs: Any) -> Any: + font = kwargs["font"] + if font.size == 21: + widths.append(font.getlength(text)) + assert font.path == fallback + return original(self, position, text, *args, **kwargs) + + monkeypatch.setattr(structured, "_text_font", choose) + monkeypatch.setattr(ImageDraw.ImageDraw, "text", capture) + answer = {"value": "I" * 100 + "日本語"} + structured._panel(answer, json.dumps(answer)) + assert len(widths) > 1 + assert max(widths) <= 402 + + +def test_json_fallback_preserves_supported_text( + monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from matplotlib.font_manager import FontEntry + + from vlm_exam.visualization import structured + + monkeypatch.setattr( + structured.font_manager.fontManager, + "ttflist", + [ + FontEntry(fname="placeholder", name="LastResortHE"), + FontEntry(fname="cjk", name="Noto Sans CJK"), + ], + ) + structured._fallbacks.cache_clear() + try: + assert structured._fallbacks() == ("cjk",) + monkeypatch.setattr( + structured.font_manager, "findfont", lambda *args: "primary" + ) + coverage = { + "primary": frozenset(map(ord, "Invoice paid ")), + "cjk": frozenset(map(ord, "日本語")), + } + monkeypatch.setattr(structured, "_coverage", coverage.__getitem__) + monkeypatch.setattr(structured, "_font", lambda path, size: path) + assert structured._text_font("Invoice paid 🚀", 21) == "primary" + assert structured._text_font("日本語😀", 21) == "cjk" + finally: + structured._fallbacks.cache_clear() + + +def test_deep_json_and_json_like_strings_stay_bounded(card_fonts: None) -> None: + from vlm_exam.visualization import structured + + nested: Any = 1 + for _ in range(2000): + nested = [nested] + fields = structured._fields({"a": 1}, {"a": nested}) + assert fields[-1].path == ("a",) + (0,) * 2000 + for response in ( + '{"a":' + "[" * 1000 + "1" + "]" * 1000 + "}", + json.dumps({"a": "[" * 10000 + "1" + "]" * 10000}), + ): + assert structured._panel({"a": "ok"}, response).size == (879, 604) From e585c857a179d0582a704e6f5b8b965ead516921 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 12 Oct 2026 18:02:23 +0200 Subject: [PATCH 3/8] Apply approved crimson palette and minimal JSON failure state --- README.md | 5 ++- src/vlm_exam/visualization/structured.py | 44 +++++++----------------- tests/test_cli_run.py | 19 +++++++--- 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index f5e19ea..3aa5774 100644 --- a/README.md +++ b/README.md @@ -466,7 +466,10 @@ shared [leaderboard directory](visualizations/leaderboards/). Structured JSON cards show one image per sample, including large responses. Small objects show all fields; larger responses add a field agreement map grouped -by JSON parent paths and prioritize mismatches in the value tiles. Purple marks +by JSON parent paths and prioritize mismatches in the value tiles. Cool crimson marks differences, with the expected value struck through above the model value. FIELD F1 uses the shared detection-score styling. Long values may be shortened with an ellipsis; full responses remain in the source result file. + +Unparseable JSON uses a centered “Could not parse JSON” message on a pale pink +background. Empty responses show “No response”; the saved FIELD F1 remains visible. diff --git a/src/vlm_exam/visualization/structured.py b/src/vlm_exam/visualization/structured.py index 117a2c2..2b2d9df 100644 --- a/src/vlm_exam/visualization/structured.py +++ b/src/vlm_exam/visualization/structured.py @@ -33,10 +33,10 @@ _HEIGHT = 604 _MISSING = object() _MAP_MATCH = "#E4E3E9" -_MAP_CHANGED = "#8253BC" +_MAP_CHANGED = "#BE254B" _CARD_MATCH = "#F3F2F5" -_CARD_CHANGED = "#F0EAF8" -_VALUE_CHANGED = "#743AC5" +_CARD_CHANGED = "#FAEDF1" +_VALUE_CHANGED = "#BE254B" @dataclass(frozen=True) @@ -312,35 +312,17 @@ def _panel(expected: Any, response: str) -> Image.Image: try: predicted, _ = parse_json(response) except (ValueError, TypeError, RecursionError): - message = "Invalid JSON" if response.strip() else "Empty response" - draw.text((14, 14), message, fill=_VALUE_CHANGED, font=_text_font(message, 21)) - raw = ( - response.replace("\n", " ↵ ").replace("\r", " ↵ ").replace("\t", " ⇥ ") - if response - else "(empty response)" - ) - preview = _wrap(raw, _WIDTH - 28, 20, 4) - preview_font = _text_font(raw, 20) - for index, line in enumerate(preview): - draw.text( - (14, 57 + index * 25), - line, - fill=theme.TEXT_PRIMARY, - font=preview_font, - ) - draw.line((0, 175, _WIDTH - 1, 175), fill=theme.DIVIDER_COLOR, width=1) - draw.text( - (14, 192), - "Expected", - fill=theme.TEXT_SECONDARY, - font=_text_font("Expected", 14), + message = "Could not parse JSON" if response.strip() else "No response" + font = _text_font(message, 22) + left, top, right, bottom = draw.textbbox((0, 0), message, font=font) + x = (_WIDTH - (right - left)) / 2 - left + y = (_HEIGHT - (bottom - top)) / 2 - top + draw.rounded_rectangle( + (x - 16, y + top - 12, x + right + 16, y + bottom + 12), + radius=5, + fill=_CARD_CHANGED, ) - reference = _fields(expected, expected) - for count in range(min(6, len(reference)), 0, -1): - tiles, bottom = _layout(reference[:count], 2, 20, top=225, limit=2) - if bottom <= _HEIGHT: - _draw_tiles(draw, tiles) - break + draw.text((x, y), message, font=font, fill=_VALUE_CHANGED) return panel fields = _fields(expected, predicted) for columns, size in ((2, 21), (2, 20), (2, 19), (3, 19), (3, 18), (3, 17)): diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index e63b34e..2711310 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -254,7 +254,7 @@ def card_fonts(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.parametrize("prediction", ["", '{"total":', '{"x":NaN}', '{"x":1,"x":2}']) -def test_invalid_json_card_keeps_reference( +def test_invalid_json_card_has_centered_failure_message( prediction: str, monkeypatch: pytest.MonkeyPatch, card_fonts: None ) -> None: from PIL import ImageDraw @@ -271,9 +271,20 @@ def capture(self: Any, position: Any, text: str, *args: Any, **kwargs: Any) -> A monkeypatch.setattr(ImageDraw.ImageDraw, "text", capture) panel = structured._panel({"total": "42.00"}, prediction) assert panel.size == (879, 604) - assert "total" in drawn and '"42.00"' in drawn - assert "Invalid JSON" in drawn if prediction else "Empty response" in drawn - assert prediction in drawn if prediction else "(empty response)" in drawn + assert drawn == ["Could not parse JSON" if prediction else "No response"] + background = panel.getpixel((0, 0)) + changed = [ + (x, y) + for y in range(panel.height) + for x in range(panel.width) + if panel.getpixel((x, y)) != background + ] + assert ( + abs((min(x for x, _ in changed) + max(x for x, _ in changed)) / 2 - 439.5) <= 1 + ) + assert ( + abs((min(y for _, y in changed) + max(y for _, y in changed)) / 2 - 301.5) <= 1 + ) @pytest.mark.parametrize("score", [0.0, 0.8, 0.95, None]) From 05a23b33c02ac08e8a60b1e582a819acefbe3d10 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 12 Oct 2026 19:30:29 +0200 Subject: [PATCH 4/8] Apply approved purple hybrid JSON card headers --- README.md | 6 +- src/vlm_exam/visualization/structured.py | 84 +++++++++++++++++++----- tests/test_cli_run.py | 49 ++++++++++++++ 3 files changed, 119 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3aa5774..a324273 100644 --- a/README.md +++ b/README.md @@ -466,10 +466,12 @@ shared [leaderboard directory](visualizations/leaderboards/). Structured JSON cards show one image per sample, including large responses. Small objects show all fields; larger responses add a field agreement map grouped -by JSON parent paths and prioritize mismatches in the value tiles. Cool crimson marks +by JSON parent paths and prioritize mismatches in the value tiles. Purple marks differences, with the expected value struck through above the model value. +The full-width JSON MAP includes total field and difference counts; the values +header identifies the struck-through Reference and purple Model. FIELD F1 uses the shared detection-score styling. Long values may be shortened with an ellipsis; full responses remain in the source result file. -Unparseable JSON uses a centered “Could not parse JSON” message on a pale pink +Unparseable JSON uses a centered “Could not parse JSON” message on a pale purple background. Empty responses show “No response”; the saved FIELD F1 remains visible. diff --git a/src/vlm_exam/visualization/structured.py b/src/vlm_exam/visualization/structured.py index 2b2d9df..9174db7 100644 --- a/src/vlm_exam/visualization/structured.py +++ b/src/vlm_exam/visualization/structured.py @@ -33,10 +33,10 @@ _HEIGHT = 604 _MISSING = object() _MAP_MATCH = "#E4E3E9" -_MAP_CHANGED = "#BE254B" +_MAP_CHANGED = "#8253BC" _CARD_MATCH = "#F3F2F5" -_CARD_CHANGED = "#FAEDF1" -_VALUE_CHANGED = "#BE254B" +_CARD_CHANGED = "#F0EAF8" +_VALUE_CHANGED = "#743AC5" @dataclass(frozen=True) @@ -224,7 +224,7 @@ def _layout( field, column * (width + 18), bottoms[column], - width, + _WIDTH - column * (width + 18) if column == columns - 1 else width, height, labels, values, @@ -306,6 +306,43 @@ def _draw_tiles(draw: ImageDraw.ImageDraw, tiles: list[_Tile]) -> None: y += tile.size + 5 +def _header( + draw: ImageDraw.ImageDraw, + title: str, + y: int, + total: int | None = None, + different: int = 0, +) -> None: + font = _font(font_manager.findfont(theme.load_fonts().bold), 17) + draw.text((0, y + 5), title, font=font, fill=theme.PANEL_LABEL_COLOR) + if total is not None: + draw.text( + (font.getlength(title) + 14, y + 6), + f"{total} fields", + font=_text_font("fields", 17), + fill=theme.TEXT_SECONDARY, + ) + label = f"{different} different" + font = _text_font(label, 17) + draw.text( + (_WIDTH - 1 - font.getlength(label), y + 6), + label, + font=font, + fill=_VALUE_CHANGED, + ) + return + font = _text_font("Reference Model", 16) + model_x = _WIDTH - 1 - font.getlength("Model") + reference_x = model_x - 20 - font.getlength("Reference") + draw.text((reference_x, y + 6), "Reference", font=font, fill=theme.TEXT_SECONDARY) + draw.line( + (reference_x, y + 16, reference_x + font.getlength("Reference"), y + 16), + fill=theme.TEXT_SECONDARY, + width=1, + ) + draw.text((model_x, y + 6), "Model", font=font, fill=_VALUE_CHANGED) + + def _panel(expected: Any, response: str) -> Image.Image: panel = Image.new("RGB", (_WIDTH, _HEIGHT), theme.BACKGROUND_COLOR) draw = ImageDraw.Draw(panel) @@ -326,30 +363,41 @@ def _panel(expected: Any, response: str) -> Image.Image: return panel fields = _fields(expected, predicted) for columns, size in ((2, 21), (2, 20), (2, 19), (3, 19), (3, 18), (3, 17)): - tiles, bottom = _layout(fields, columns, size) + tiles, bottom = _layout(fields, columns, size, top=36) if bottom <= _HEIGHT: + _header(draw, "FIELD VALUES", 0) _draw_tiles(draw, tiles) return panel if len(fields) <= 2: - tiles, _ = _layout(fields, 1, 20, limit=9 if len(fields) == 1 else 3) + tiles, _ = _layout(fields, 1, 20, top=36, limit=8 if len(fields) == 1 else 3) + _header(draw, "FIELD VALUES", 0) _draw_tiles(draw, tiles) return panel - columns = 40 if len(fields) > 160 else 24 if len(fields) > 72 else 16 - rows = math.ceil(len(fields) / columns) - bottom = int(rows * min(16, 208 / rows) + 4) - cells = _map_layout(fields, bottom + 10) + map_height = min(180, max(65, math.ceil(len(fields) / 40) * 14)) + values_y = 36 + map_height + 16 + ranked = sorted(fields, key=lambda field: field.matches) + for count in range(min(12, len(fields)), 0, -1): + tiles, bottom = _layout(ranked[:count], 2, 20, top=values_y + 36, limit=2) + if bottom <= _HEIGHT: + break + _header( + draw, "JSON MAP", 0, len(fields), sum(not field.matches for field in fields) + ) + cells = _map_layout(fields, map_height) + scale = _WIDTH / max(x + size for x, _, size, _ in cells) for x, y, size, matches in cells: draw.rectangle( - (x, y, x + size - 1, y + size - 1), + ( + round(x * scale), + y + 36, + round((x + size) * scale) - 1, + y + 36 + size - 1, + ), fill=_MAP_MATCH if matches else _MAP_CHANGED, ) - draw.line((0, bottom + 17, _WIDTH - 1, bottom + 17), fill="#E2E2E9", width=1) - ranked = sorted(fields, key=lambda field: field.matches) - for count in range(min(12, len(fields) - 1), 0, -1): - tiles, height = _layout(ranked[:count], 2, 20, bottom + 35, 2) - if height <= _HEIGHT: - _draw_tiles(draw, tiles) - break + draw.line((0, values_y - 10, _WIDTH - 1, values_y - 10), fill="#E2E2E9", width=1) + _header(draw, "SELECTED VALUES", values_y) + _draw_tiles(draw, tiles) return panel diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index 2711310..de450f3 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -383,6 +383,55 @@ def cells(fields: list[Any], height: int) -> list[Any]: ) +@pytest.mark.parametrize("expected", [{}, [], {"long": "x" * 20000}]) +def test_json_hybrid_small_values_omit_map( + expected: Any, monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from vlm_exam.visualization import structured + + labels: list[str] = [] + original = structured._header + + def header(draw: Any, title: str, y: int, *args: Any) -> None: + labels.append(title) + original(draw, title, y, *args) + + monkeypatch.setattr(structured, "_header", header) + structured._panel(expected, json.dumps(expected)) + assert labels == ["FIELD VALUES"] + + +def test_json_hybrid_map_counts_and_full_width( + monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from vlm_exam.visualization import structured + + headers: list[tuple[Any, ...]] = [] + rectangles: list[tuple[int, ...]] = [] + original = structured._header + from PIL import ImageDraw + + rectangle = ImageDraw.ImageDraw.rectangle + + def header(draw: Any, title: str, y: int, *args: Any) -> None: + headers.append((title, *args)) + original(draw, title, y, *args) + + def record(draw: Any, box: Any, *args: Any, **kwargs: Any) -> None: + if kwargs.get("fill") in (structured._MAP_MATCH, structured._MAP_CHANGED): + rectangles.append(box) + rectangle(draw, box, *args, **kwargs) + + monkeypatch.setattr(structured, "_header", header) + monkeypatch.setattr(ImageDraw.ImageDraw, "rectangle", record) + expected = {"rows": list(range(620))} + predicted = {"rows": list(range(619)) + [-1]} + structured._panel(expected, json.dumps(predicted)) + assert headers == [("JSON MAP", 620, 1), ("SELECTED VALUES",)] + assert min(box[0] for box in rectangles) == 0 + assert max(box[2] for box in rectangles) == 878 + + @pytest.mark.parametrize("fields", [1, 2, 4]) def test_long_json_strings_remain_within_card( fields: int, monkeypatch: pytest.MonkeyPatch, card_fonts: None From f3c9ac46894eba1600967d27fe27ed4a5ba97c7c Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 12 Oct 2026 20:11:28 +0200 Subject: [PATCH 5/8] Apply approved text-only transcription cards --- README.md | 6 + src/vlm_exam/visualization/text.py | 53 ++++++-- src/vlm_exam/visualization/transcription.py | 128 ++++++++++++++++++++ tests/test_cli_run.py | 47 +++++++ 4 files changed, 221 insertions(+), 13 deletions(-) create mode 100644 src/vlm_exam/visualization/transcription.py diff --git a/README.md b/README.md index a324273..6f7fbaa 100644 --- a/README.md +++ b/README.md @@ -475,3 +475,9 @@ with an ellipsis; full responses remain in the source result file. Unparseable JSON uses a centered “Could not parse JSON” message on a pale purple background. Empty responses show “No response”; the saved FIELD F1 remains visible. + +Transcription cards use a single text-only panel with 18 px inline comparisons, +muted struck reference text and purple model differences. Original line breaks +are preserved; long lines wrap and overflow ends with an ellipsis. There is no +word map or pagination. The saved full-response score remains visible, labeled +Character Similarity or Exact Match according to the recorded scoring profile. diff --git a/src/vlm_exam/visualization/text.py b/src/vlm_exam/visualization/text.py index 5ae3a85..4ddf5d9 100644 --- a/src/vlm_exam/visualization/text.py +++ b/src/vlm_exam/visualization/text.py @@ -30,10 +30,10 @@ from vlm_exam.visualization import theme from vlm_exam.visualization.cases import ( plot_qa_card, - plot_transcription_card, ) from vlm_exam.visualization.detection import region_diff_image from vlm_exam.visualization.structured import draw_structured_panel +from vlm_exam.visualization.transcription import draw_transcription_panel def _frame( @@ -349,18 +349,45 @@ def plot_text_cards( image, sample, result, task, model, config, uploaded_size ) ] - if sample.category == "transcription" and sample.scoring_profile != "exact": - return [ - plot_transcription_card( - image, - sample.expected, - result.predicted, - result.metadata["score"], - model, - config, - normalize=False, - ) - ] + if sample.category == "transcription": + figure, rail = _frame(image, model, config, "TRANSCRIPTION") + draw_transcription_panel(rail, sample.expected, result.predicted) + fonts = theme.load_fonts() + exact = sample.scoring_profile == "exact" + rail.plot( + [0, 1], [0.262, 0.262], color=theme.DIVIDER_COLOR, lw=1, clip_on=False + ) + rail.text( + 0.5, + 0.212, + "EXACT MATCH" if exact else "CHARACTER SIMILARITY", + fontsize=9.5, + ha="center", + va="center", + color=theme.PANEL_LABEL_COLOR, + font=fonts.bold, + ) + score = result.metadata.get("score") + scored = ( + type(score) in (int, float) and math.isfinite(score) and 0 <= score <= 1 + ) + rail.text( + 0.5, + 0.142, + f"{score * 100:.1f}%" if scored else "—", + fontsize=26, + ha="center", + va="center", + font=fonts.display, + color=theme.score_color(score) if scored else theme.TEXT_SECONDARY, + ) + theme.draw_brand_footer( + rail, + "evaluated via exact match" + if exact + else "evaluated via character similarity", + ) + return [figure] def display(value: str) -> str: return normalize(value).replace("\n", " ↵ ").replace("\t", " ⇥ ") diff --git a/src/vlm_exam/visualization/transcription.py b/src/vlm_exam/visualization/transcription.py new file mode 100644 index 0000000..ce38f08 --- /dev/null +++ b/src/vlm_exam/visualization/transcription.py @@ -0,0 +1,128 @@ +# Copyright 2026 Roboflow, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import difflib +import re + +import matplotlib.pyplot as plt +from PIL import Image, ImageDraw + +from vlm_exam.tasks.text_scoring import normalize +from vlm_exam.visualization import structured, theme + +_WIDTH = 879 +_HEIGHT = 604 + + +def _flow(draw: ImageDraw.ImageDraw, runs: list[tuple[str, str]]) -> bool: + x = 0 + y = 36 + size = 18 + line = size + 5 + for value, kind in runs: + for token in re.findall(r"\n|[^\S\n]+|\S+", value): + if token == "\n": + x = 0 + y += line + continue + token = token.expandtabs(4) + parts = ( + list(token) + if structured._text_font(token, size).getlength(token) > _WIDTH + else [token] + ) + for part in parts: + f = structured._text_font(part, size) + width = f.getlength(part) + if x + width > _WIDTH: + x = 0 + y += line + if part.isspace(): + continue + if y > _HEIGHT - size - 30: + return True + color = ( + structured._VALUE_CHANGED + if kind == "insert" + else theme.TEXT_SECONDARY + if kind == "delete" + else theme.TEXT_PRIMARY + ) + draw.text((x, y), part, font=f, fill=color) + if kind == "delete" and not part.isspace(): + box = draw.textbbox((x, y), part, font=f) + draw.line( + (x, (box[1] + box[3]) / 2, x + width, (box[1] + box[3]) / 2), + fill=theme.TEXT_SECONDARY, + ) + x += width + return False + + +def _panel(expected: str, predicted: str) -> Image.Image: + a, b = normalize(expected), normalize(predicted) + if not b: + return structured._panel({}, "") + panel = Image.new("RGB", (_WIDTH, _HEIGHT), theme.BACKGROUND_COLOR) + draw = ImageDraw.Draw(panel) + tokens_a = re.findall(r"\n|[^\S\n]+|\S+", a) + tokens_b = re.findall(r"\n|[^\S\n]+|\S+", b) + runs = [] + for kind, i, j, k, end in difflib.SequenceMatcher( + None, tokens_a, tokens_b, autojunk=False + ).get_opcodes(): + if kind == "equal": + runs.append(("".join(tokens_b[k:end]), "equal")) + else: + if i < j: + runs.append(("".join(tokens_a[i:j]), "delete")) + if ( + i < j + and k < end + and not tokens_a[j - 1].isspace() + and not tokens_b[k].isspace() + ): + runs.append((" ", "equal")) + if k < end: + runs.append(("".join(tokens_b[k:end]), "insert")) + structured._header(draw, "TRANSCRIPTION", 0) + if _flow(draw, runs): + draw.text( + (0, _HEIGHT - 26), + "…", + font=structured._text_font("…", 20), + fill=theme.TEXT_SECONDARY, + ) + return panel + + +def draw_transcription_panel(rail: plt.Axes, expected: str, predicted: str) -> None: + """Draw a bounded inline comparison preserving original line breaks. + + Args: + rail: Native card results axes. + expected: Reference transcription. + predicted: Saved model response. + """ + rail.imshow( + _panel(expected, predicted), + extent=(0, 1, 0.279, 0.869), + aspect="auto", + interpolation="nearest", + zorder=0, + ) + rail.set_xlim(0, 1) + rail.set_ylim(0, 1) diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index de450f3..b678654 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -591,3 +591,50 @@ def test_deep_json_and_json_like_strings_stay_bounded(card_fonts: None) -> None: json.dumps({"a": "[" * 10000 + "1" + "]" * 10000}), ): assert structured._panel({"a": "ok"}, response).size == (879, 604) + + +@pytest.mark.parametrize("content", ["ELL\nJAIL\nVOW", "a\n\nb", "日本語\nInvoice"]) +def test_transcription_preserves_line_breaks( + content: str, monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from PIL import ImageDraw + + from vlm_exam.visualization import transcription + + positions = [] + original = ImageDraw.ImageDraw.text + + def capture(self: Any, position: Any, value: str, *args: Any, **kwargs: Any) -> Any: + if value in content.split("\n") and value: + positions.append(position) + return original(self, position, value, *args, **kwargs) + + monkeypatch.setattr(ImageDraw.ImageDraw, "text", capture) + transcription._panel(content, content) + assert [pos[1] for pos in positions] == [ + 36 + index * 23 for index, line in enumerate(content.split("\n")) if line + ] + assert all(pos[0] == 0 for pos in positions) + + +def test_transcription_overflow_is_bounded( + card_fonts: None, monkeypatch: pytest.MonkeyPatch +) -> None: + from PIL import ImageDraw + + from vlm_exam.visualization import transcription + + drawn = [] + original = ImageDraw.ImageDraw.text + + def capture(self: Any, position: Any, value: str, *args: Any, **kwargs: Any) -> Any: + drawn.append((position, value)) + return original(self, position, value, *args, **kwargs) + + monkeypatch.setattr(ImageDraw.ImageDraw, "text", capture) + image = transcription._panel("word\n" * 1000, "word\n" * 1000) + assert image.size == (879, 604) + assert drawn[-1][1] == "…" + assert len(drawn) < 40 + assert all(0 <= position[1] < 604 for position, _ in drawn) + assert not any("MAP" in value for _, value in drawn) From 1e17fbaf63f875f1779220862f51dbc204fff3b4 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 12 Oct 2026 20:52:58 +0200 Subject: [PATCH 6/8] Apply centered single-value extraction design --- README.md | 5 + src/vlm_exam/visualization/single_value.py | 133 +++++++++++++++++++++ src/vlm_exam/visualization/text.py | 42 +++---- tests/test_cli_run.py | 40 +++++++ 4 files changed, 193 insertions(+), 27 deletions(-) create mode 100644 src/vlm_exam/visualization/single_value.py diff --git a/README.md b/README.md index 6f7fbaa..130c3a4 100644 --- a/README.md +++ b/README.md @@ -481,3 +481,8 @@ muted struck reference text and purple model differences. Original line breaks are preserved; long lines wrap and overflow ends with an ellipsis. There is no word map or pagination. The saved full-response score remains visible, labeled Character Similarity or Exact Match according to the recorded scoring profile. + +Single-value cards retain the question above a centered 24 px answer comparison. +Correct values appear once in neutral text; differences show the muted struck +reference above the purple model value. The saved Exact Match score uses the +same bottom metric position as the other text cards. diff --git a/src/vlm_exam/visualization/single_value.py b/src/vlm_exam/visualization/single_value.py new file mode 100644 index 0000000..63006be --- /dev/null +++ b/src/vlm_exam/visualization/single_value.py @@ -0,0 +1,133 @@ +# Copyright 2026 Roboflow, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json + +import matplotlib.pyplot as plt +from matplotlib import font_manager +from PIL import Image, ImageDraw + +from vlm_exam.tasks.text_scoring import normalize +from vlm_exam.visualization import structured, theme + +_WIDTH = 879 +_HEIGHT = 604 + + +def _question_lines(value: str) -> list[str]: + font = structured._text_font(value, 22) + lines = [] + current = "" + for word in value.split(): + candidate = f"{current} {word}".strip() + if current and font.getlength(candidate) > _WIDTH - 8: + lines.append(current) + current = "" + pieces = ( + structured._wrap(word, _WIDTH - 8, 22, 100) + if font.getlength(word) > _WIDTH - 8 + else [word] + ) + for piece in pieces[:-1]: + lines.append(piece) + current = f"{current} {pieces[-1]}".strip() + if current: + lines.append(current) + if len(lines) > 7: + lines = lines[:7] + while lines[-1] and font.getlength(lines[-1] + "…") > _WIDTH - 8: + lines[-1] = lines[-1][:-1] + lines[-1] += "…" + return lines + + +def _panel(question: str, expected: str, predicted: str) -> Image.Image: + panel = Image.new("RGB", (_WIDTH, _HEIGHT), theme.BACKGROUND_COLOR) + draw = ImageDraw.Draw(panel) + try: + parsed = json.loads(question) + if isinstance(parsed, dict): + question = str(parsed.get("question", question)) + except (ValueError, TypeError): + pass + bold = structured._font(font_manager.findfont(theme.load_fonts().bold), 17) + draw.text((0, 5), "QUESTION", font=bold, fill=theme.PANEL_LABEL_COLOR) + font = structured._text_font(question, 22) + lines = _question_lines(question) + for index, line in enumerate(lines): + draw.text((0, 43 + index * 29), line, font=font, fill=theme.TEXT_PRIMARY) + rule = 43 + len(lines) * 29 + 25 + draw.line((0, rule, _WIDTH - 1, rule), fill="#E2E2E9", width=1) + structured._header(draw, "VALUE", rule + 18) + expected, predicted = normalize(expected), normalize(predicted) + same = expected == predicted + items = ( + [(predicted or "No response", theme.TEXT_PRIMARY, False)] + if same + else [ + (expected or "Empty reference", theme.TEXT_SECONDARY, True), + (predicted or "No response", structured._VALUE_CHANGED, False), + ] + ) + available_top = rule + 70 + gap = 0 if same else 18 + limit = max(1, min(4, (_HEIGHT - available_top - gap) // (31 * len(items)))) + laid = [ + ( + structured._wrap(value, _WIDTH - 40, 24, limit), + structured._text_font(value, 24), + color, + strike, + ) + for value, color, strike in items + ] + height = sum(len(lines) * 31 for lines, _, _, _ in laid) + gap + y = available_top + max(0, (_HEIGHT - available_top - height) / 2) + for lines, font, color, strike in laid: + for line in lines: + x = (_WIDTH - font.getlength(line)) / 2 + draw.text((x, y), line, font=font, fill=color) + if strike: + bounds = draw.textbbox((x, y), line, font=font) + middle = (bounds[1] + bounds[3]) / 2 + draw.line( + (x, middle, x + font.getlength(line), middle), fill=color, width=1 + ) + y += 31 + y += 18 + return panel + + +def draw_single_value_panel( + rail: plt.Axes, question: str, expected: str, predicted: str +) -> None: + """Draw the question and centered single-value comparison. + + Args: + rail: Native card results axes. + question: Dataset extraction question. + expected: Reference value. + predicted: Saved model response. + """ + rail.imshow( + _panel(question, expected, predicted), + extent=(0, 1, 0.279, 0.869), + aspect="auto", + interpolation="nearest", + zorder=0, + ) + rail.set_xlim(0, 1) + rail.set_ylim(0, 1) diff --git a/src/vlm_exam/visualization/text.py b/src/vlm_exam/visualization/text.py index 4ddf5d9..64d084c 100644 --- a/src/vlm_exam/visualization/text.py +++ b/src/vlm_exam/visualization/text.py @@ -26,12 +26,10 @@ from vlm_exam.config import VisualizationConfig from vlm_exam.results import SampleResult from vlm_exam.tasks.text import TextSample, TextTask -from vlm_exam.tasks.text_scoring import match_regions, normalize, parse_json +from vlm_exam.tasks.text_scoring import match_regions, parse_json from vlm_exam.visualization import theme -from vlm_exam.visualization.cases import ( - plot_qa_card, -) from vlm_exam.visualization.detection import region_diff_image +from vlm_exam.visualization.single_value import draw_single_value_panel from vlm_exam.visualization.structured import draw_structured_panel from vlm_exam.visualization.transcription import draw_transcription_panel @@ -349,11 +347,19 @@ def plot_text_cards( image, sample, result, task, model, config, uploaded_size ) ] - if sample.category == "transcription": - figure, rail = _frame(image, model, config, "TRANSCRIPTION") - draw_transcription_panel(rail, sample.expected, result.predicted) + if sample.category in ("transcription", "single_string"): + single_value = sample.category == "single_string" + figure, rail = _frame( + image, model, config, "DATA EXTRACTION" if single_value else "TRANSCRIPTION" + ) + if single_value: + draw_single_value_panel( + rail, sample.question, sample.expected, result.predicted + ) + else: + draw_transcription_panel(rail, sample.expected, result.predicted) fonts = theme.load_fonts() - exact = sample.scoring_profile == "exact" + exact = single_value or sample.scoring_profile == "exact" rail.plot( [0, 1], [0.262, 0.262], color=theme.DIVIDER_COLOR, lw=1, clip_on=False ) @@ -389,22 +395,4 @@ def plot_text_cards( ) return [figure] - def display(value: str) -> str: - return normalize(value).replace("\n", " ↵ ").replace("\t", " ⇥ ") - - figure = plot_qa_card( - image, - sample.question, - display(sample.expected), - display(result.predicted), - result.correct, - model, - config, - "DATA EXTRACTION", - match_method="strict", - normalize=False, - ) - rail = figure.axes[1] - edge = theme.draw_legend_chip(rail, 1, 0.797, "+ model", theme.FAILURE_TEXT_COLOR) - theme.draw_legend_chip(rail, edge, 0.797, "− expected", theme.SUCCESS_TEXT_COLOR) - return [figure] + raise ValueError(f"Unsupported text category: {sample.category}") diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index b678654..45c4c34 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -638,3 +638,43 @@ def capture(self: Any, position: Any, value: str, *args: Any, **kwargs: Any) -> assert len(drawn) < 40 assert all(0 <= position[1] < 604 for position, _ in drawn) assert not any("MAP" in value for _, value in drawn) + + +@pytest.mark.parametrize("same", [True, False]) +def test_single_value_comparison_is_bounded( + same: bool, monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from PIL import ImageDraw + + from vlm_exam.visualization import single_value + + drawn = [] + original = ImageDraw.ImageDraw.text + + def capture(self: Any, position: Any, value: str, *args: Any, **kwargs: Any) -> Any: + drawn.append((position, value, kwargs.get("font"))) + return original(self, position, value, *args, **kwargs) + + monkeypatch.setattr(ImageDraw.ImageDraw, "text", capture) + expected = "ABC" * 2000 + predicted = expected if same else "XYZ" * 2000 + panel = single_value._panel("Read " * 1000, expected, predicted) + assert panel.size == (879, 604) + assert any("…" in value for _, value, _ in drawn) + assert len(drawn) < 20 + assert all(0 <= position[1] < 580 for position, _, _ in drawn) + assert all( + position[0] + font.getlength(value) <= 880 for position, value, font in drawn + ) + + +def test_single_value_question_wraps_at_words(card_fonts: None) -> None: + from vlm_exam.visualization import single_value + + question = ( + "Read the letters and numbers imprinted on the pill in this image. " + "Return only the text, preserving capitalization and separating lines " + "with a single space. Do not explain." + ) + assert " ".join(single_value._question_lines(question)) == question + assert len(single_value._question_lines("X" * 1000)) > 1 From 3d89c74f67e4f2e60bd1b6e51473bd85de489cb3 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 12 Oct 2026 21:08:30 +0200 Subject: [PATCH 7/8] Fix multiline value layout and invisible whitespace differences --- README.md | 4 ++ src/vlm_exam/visualization/single_value.py | 4 ++ src/vlm_exam/visualization/transcription.py | 14 ++++++- tests/test_cli_run.py | 42 +++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 130c3a4..6cc0ad8 100644 --- a/README.md +++ b/README.md @@ -486,3 +486,7 @@ Single-value cards retain the question above a centered 24 px answer comparison. Correct values appear once in neutral text; differences show the muted struck reference above the purple model value. The saved Exact Match score uses the same bottom metric position as the other text cards. + +Single-value comparisons display embedded newlines and tabs as visible markers +before measuring the text. Transcription keeps line breaks and marks only changed +whitespace (newline, tab or space), using the same insertion/deletion colors. diff --git a/src/vlm_exam/visualization/single_value.py b/src/vlm_exam/visualization/single_value.py index 63006be..5819633 100644 --- a/src/vlm_exam/visualization/single_value.py +++ b/src/vlm_exam/visualization/single_value.py @@ -82,6 +82,10 @@ def _panel(question: str, expected: str, predicted: str) -> Image.Image: (predicted or "No response", structured._VALUE_CHANGED, False), ] ) + items = [ + (value.replace("\n", " ↵ ").replace("\t", " ⇥ "), color, strike) + for value, color, strike in items + ] available_top = rule + 70 gap = 0 if same else 18 limit = max(1, min(4, (_HEIGHT - available_top - gap) // (31 * len(items)))) diff --git a/src/vlm_exam/visualization/transcription.py b/src/vlm_exam/visualization/transcription.py index ce38f08..0a92a2c 100644 --- a/src/vlm_exam/visualization/transcription.py +++ b/src/vlm_exam/visualization/transcription.py @@ -34,11 +34,18 @@ def _flow(draw: ImageDraw.ImageDraw, runs: list[tuple[str, str]]) -> bool: line = size + 5 for value, kind in runs: for token in re.findall(r"\n|[^\S\n]+|\S+", value): - if token == "\n": + newline = token == "\n" + if newline and kind == "equal": x = 0 y += line continue - token = token.expandtabs(4) + if kind != "equal" and token.isspace(): + token = "".join( + "↵" if char == "\n" else "⇥" if char == "\t" else "·" + for char in token + ) + else: + token = token.expandtabs(4) parts = ( list(token) if structured._text_font(token, size).getlength(token) > _WIDTH @@ -69,6 +76,9 @@ def _flow(draw: ImageDraw.ImageDraw, runs: list[tuple[str, str]]) -> bool: fill=theme.TEXT_SECONDARY, ) x += width + if newline: + x = 0 + y += line return False diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index 45c4c34..3a3fca4 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -678,3 +678,45 @@ def test_single_value_question_wraps_at_words(card_fonts: None) -> None: ) assert " ".join(single_value._question_lines(question)) == question assert len(single_value._question_lines("X" * 1000)) > 1 + + +@pytest.mark.parametrize("predicted", ["a\n\nb", "a b", "a\tb", "a b"]) +def test_transcription_whitespace_differences_are_visible( + predicted: str, card_fonts: None +) -> None: + from PIL import ImageChops + + from vlm_exam.visualization import transcription + + expected = "a\nb" + comparison = transcription._panel(expected, predicted) + identical = transcription._panel(predicted, predicted) + assert ImageChops.difference(comparison, identical).getbbox() is not None + + +@pytest.mark.parametrize("whitespace, marker", [("\n", "↵"), ("\t", "⇥")]) +def test_single_value_whitespace_is_measured_as_visible_text( + whitespace: str, marker: str, monkeypatch: pytest.MonkeyPatch, card_fonts: None +) -> None: + from PIL import ImageDraw + + from vlm_exam.visualization import single_value + + bounds = [] + painted = [] + original = ImageDraw.ImageDraw.text + + def capture(self: Any, position: Any, value: str, *args: Any, **kwargs: Any) -> Any: + painted.append(value) + if kwargs.get("font").size == 24: + bounds.append(self.textbbox(position, value, font=kwargs["font"])) + return original(self, position, value, *args, **kwargs) + + monkeypatch.setattr(ImageDraw.ImageDraw, "text", capture) + single_value._panel( + "Read the meter.", "59150", whitespace.join(["Dial one explanation"] * 100) + ) + assert any(marker in value for value in painted) + assert all("\n" not in value and "\t" not in value for value in painted) + assert all(0 <= top < bottom <= 604 for _, top, _, bottom in bounds) + assert all(a[3] <= b[1] for a, b in zip(bounds, bounds[1:])) From 85f8df405b90edc4ac90aae937903657898347bb Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Tue, 13 Oct 2026 00:12:10 +0200 Subject: [PATCH 8/8] Align localization cards and exclude unmatched region crops --- src/vlm_exam/visualization/structured.py | 28 +++-- src/vlm_exam/visualization/text.py | 132 ++++++++++++++--------- tests/test_cli_run.py | 68 ++++++++++++ 3 files changed, 164 insertions(+), 64 deletions(-) diff --git a/src/vlm_exam/visualization/structured.py b/src/vlm_exam/visualization/structured.py index 9174db7..f49a742 100644 --- a/src/vlm_exam/visualization/structured.py +++ b/src/vlm_exam/visualization/structured.py @@ -343,6 +343,22 @@ def _header( draw.text((model_x, y + 6), "Model", font=font, fill=_VALUE_CHANGED) +def _message_panel(message: str) -> Image.Image: + panel = Image.new("RGB", (_WIDTH, _HEIGHT), theme.BACKGROUND_COLOR) + draw = ImageDraw.Draw(panel) + font = _text_font(message, 22) + left, top, right, bottom = draw.textbbox((0, 0), message, font=font) + x = (_WIDTH - (right - left)) / 2 - left + y = (_HEIGHT - (bottom - top)) / 2 - top + draw.rounded_rectangle( + (x - 16, y + top - 12, x + right + 16, y + bottom + 12), + radius=5, + fill=_CARD_CHANGED, + ) + draw.text((x, y), message, font=font, fill=_VALUE_CHANGED) + return panel + + def _panel(expected: Any, response: str) -> Image.Image: panel = Image.new("RGB", (_WIDTH, _HEIGHT), theme.BACKGROUND_COLOR) draw = ImageDraw.Draw(panel) @@ -350,17 +366,7 @@ def _panel(expected: Any, response: str) -> Image.Image: predicted, _ = parse_json(response) except (ValueError, TypeError, RecursionError): message = "Could not parse JSON" if response.strip() else "No response" - font = _text_font(message, 22) - left, top, right, bottom = draw.textbbox((0, 0), message, font=font) - x = (_WIDTH - (right - left)) / 2 - left - y = (_HEIGHT - (bottom - top)) / 2 - top - draw.rounded_rectangle( - (x - 16, y + top - 12, x + right + 16, y + bottom + 12), - radius=5, - fill=_CARD_CHANGED, - ) - draw.text((x, y), message, font=font, fill=_VALUE_CHANGED) - return panel + return _message_panel(message) fields = _fields(expected, predicted) for columns, size in ((2, 21), (2, 20), (2, 19), (3, 19), (3, 18), (3, 17)): tiles, bottom = _layout(fields, columns, size, top=36) diff --git a/src/vlm_exam/visualization/text.py b/src/vlm_exam/visualization/text.py index 64d084c..efdf248 100644 --- a/src/vlm_exam/visualization/text.py +++ b/src/vlm_exam/visualization/text.py @@ -30,7 +30,7 @@ from vlm_exam.visualization import theme from vlm_exam.visualization.detection import region_diff_image from vlm_exam.visualization.single_value import draw_single_value_panel -from vlm_exam.visualization.structured import draw_structured_panel +from vlm_exam.visualization.structured import _message_panel, draw_structured_panel from vlm_exam.visualization.transcription import draw_transcription_panel @@ -110,28 +110,6 @@ def _region_rows( "correct": (a in used_reference), } ) - used_reference = {a for a, _ in paired} - used_predictions = {b for _, b in paired} - rows.extend( - { - "bbox": region["bbox"], - "expected": region["text"], - "predicted": "[missing region]", - "correct": False, - } - for index, region in enumerate(reference) - if index not in used_reference - ) - rows.extend( - { - "bbox": region["bbox"], - "expected": "[extra region]", - "predicted": region["text"], - "correct": False, - } - for index, region in enumerate(predictions) - if index not in used_predictions and region is not None - ) return sorted( rows, key=lambda row: ( @@ -152,11 +130,13 @@ def _localization_card( config: VisualizationConfig, uploaded_size: tuple[int, int] | None, ) -> plt.Figure: + parse_failed = False try: parsed, _ = parse_json(result.predicted) predictions = task.parse_regions(parsed, sample, uploaded_size) except (ValueError, TypeError, KeyError, RecursionError): predictions = [None] + parse_failed = True reference = sample.answer rows = _region_rows(reference, predictions) @@ -189,7 +169,7 @@ def boxes(regions: list[dict[str, Any] | None]) -> sv.Detections: selected = rows[: 12 if columns == 3 else 8 if columns == 2 else 3] gutter = 0.045 tile_width = (1 - gutter * (columns - 1)) / columns - font_size = 10 if columns == 3 else 11 + font_size = 9.6 line_height = font_size * 1.25 / 72 / rail_height crops = [] for row in selected: @@ -213,11 +193,11 @@ def boxes(regions: list[dict[str, Any] | None]) -> sv.Detections: tiles = [] for row, crop in zip(selected, crops): values = ( - [("", row["predicted"], theme.SUCCESS_TEXT_COLOR)] + [("", row["predicted"], theme.TEXT_PRIMARY)] if row["correct"] else [ - ("− ", row["expected"], theme.SUCCESS_TEXT_COLOR), - ("+ ", row["predicted"], theme.FAILURE_TEXT_COLOR), + ("", row["expected"], theme.TEXT_SECONDARY), + ("", row["predicted"], "#743AC5"), ] ) blocks = [ @@ -225,6 +205,8 @@ def boxes(regions: list[dict[str, Any] | None]) -> sv.Detections: textwrap.fill( prefix + ("null" if value is None else str(value)), width=55 if columns == 1 else 29 if columns == 2 else 21, + max_lines=6, + placeholder="…", ), color, ) @@ -245,7 +227,7 @@ def boxes(regions: list[dict[str, Any] | None]) -> sv.Detections: groups = [tiles[i : i + columns] for i in range(0, len(tiles), columns)] heights = [max(t[2] + 0.013 + t[4] for t in group) for group in groups] gap = 0.040 if columns == 3 else 0.05 - lower = 0.19 if len(rows) > len(selected) else 0.13 + lower = 0.34 available = 0.85 - lower while groups and sum(heights) + gap * (len(groups) - 1) > available: groups.pop() @@ -267,46 +249,90 @@ def boxes(regions: list[dict[str, Any] | None]) -> sv.Detections: axes.set_xticks([]) axes.set_yticks([]) for spine in axes.spines.values(): - spine.set_color(theme.ROBOFLOW_PURPLE) - spine.set_linewidth(1.6) + spine.set_color("#743AC5") + spine.set_linewidth(1.2) y = crop_top - height - 0.013 for text, color in blocks: - rail.text( - center, - y, - text, - font=fonts.mono, - fontsize=font_size, - color=color, - va="top", - ha="center", - multialignment="center", - parse_math=False, - linespacing=1.25, - ) + for line_index, line in enumerate(text.split("\n")): + line_y = y - line_index * line_height + artist = rail.text( + center, + line_y, + line, + font=fonts.regular, + fontsize=font_size, + color=color, + va="top", + ha="center", + parse_math=False, + ) + if color == theme.TEXT_SECONDARY: + figure.canvas.draw() + bounds = artist.get_window_extent( + figure.canvas.get_renderer() + ) + points = rail.transData.inverted().transform( + [ + (bounds.x0, (bounds.y0 + bounds.y1) / 2), + (bounds.x1, (bounds.y0 + bounds.y1) / 2), + ] + ) + rail.plot(points[:, 0], points[:, 1], color=color, lw=0.6) y -= (text.count("\n") + 1) * line_height + 0.004 top -= group_height + gap if len(rows) > shown: rail.text( 0.5, - 0.14, + 0.30, f"+ {len(rows) - shown} more regions", font=fonts.medium, - fontsize=11, + fontsize=9.6, color=theme.TEXT_SECONDARY, va="top", ha="center", ) else: - rail.text( - 0.5, - 0.5, - "No valid text regions", - font=fonts.medium, - fontsize=14, - color=theme.TEXT_SECONDARY, - ha="center", + message = ( + "No response" + if not result.predicted.strip() + else "Could not parse text regions" + if parse_failed or (predictions and all(p is None for p in predictions)) + else "No text regions detected" + if not predictions + else "No matching text regions" ) + rail.imshow( + _message_panel(message), + extent=(0, 1, 0.279, 0.869), + aspect="auto", + interpolation="nearest", + zorder=0, + ) + rail.set_xlim(0, 1) + rail.set_ylim(0, 1) + score = result.metadata.get("score") + scored = type(score) in (int, float) and math.isfinite(score) and 0 <= score <= 1 + rail.plot([0, 1], [0.262, 0.262], color=theme.DIVIDER_COLOR, lw=1, clip_on=False) + rail.text( + 0.5, + 0.212, + "BOX + TEXT F1", + fontsize=9.5, + ha="center", + va="center", + color=theme.PANEL_LABEL_COLOR, + font=fonts.bold, + ) + rail.text( + 0.5, + 0.142, + f"{score * 100:.1f}%" if scored else "—", + fontsize=26, + ha="center", + va="center", + font=fonts.display, + color=theme.score_color(score) if scored else theme.TEXT_SECONDARY, + ) invalid = sum(region is None for region in predictions) theme.draw_brand_footer( rail, diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index 3a3fca4..35e807d 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -720,3 +720,71 @@ def capture(self: Any, position: Any, value: str, *args: Any, **kwargs: Any) -> assert all("\n" not in value and "\t" not in value for value in painted) assert all(0 <= top < bottom <= 604 for _, top, _, bottom in bounds) assert all(a[3] <= b[1] for a, b in zip(bounds, bounds[1:])) + + +@pytest.mark.parametrize("overlap, count", [(0, 0), (5, 0), (6, 1), (10, 1)]) +def test_localization_crops_require_spatial_match(overlap: int, count: int) -> None: + from vlm_exam.visualization.text import _region_rows + + reference = [{"bbox": [0, 0, 10, 10], "text": "same"}] + predictions = [{"bbox": [0, 0, overlap, 10], "text": "same"}, None] + assert len(_region_rows(reference, predictions)) == count + assert _region_rows(reference, []) == [] + assert _region_rows([], predictions) == [] + rows = _region_rows(reference, [{"bbox": [0, 0, 10, 10], "text": "wrong"}]) + assert len(rows) == 1 + assert rows[0]["expected"] == "same" + assert rows[0]["predicted"] == "wrong" + assert not rows[0]["correct"] + + +@pytest.mark.parametrize( + "response, predictions, message", + [ + ("", [], "No response"), + ("{", [], "Could not parse text regions"), + ("[]", [], "No text regions detected"), + ("[{}]", [None], "Could not parse text regions"), + ( + "[{}]", + [{"bbox": [15, 15, 20, 20], "text": "same"}], + "No matching text regions", + ), + ], +) +def test_localization_failure_matches_json_style( + response: str, + predictions: list[Any], + message: str, + monkeypatch: pytest.MonkeyPatch, + card_fonts: None, +) -> None: + from types import SimpleNamespace + + import matplotlib.pyplot as plt + import numpy as np + from PIL import Image + + from vlm_exam.config import DisplayConfig, ModelDisplay + from vlm_exam.visualization import structured, text + + config = DisplayConfig( + labs={"lab": LabConfig("Lab", "#000000", "")}, + models={"test": ModelDisplay("Test", "lab")}, + ) + task = SimpleNamespace(parse_regions=lambda *args: predictions) + sample = SimpleNamespace(answer=[{"bbox": [0, 0, 10, 10], "text": "same"}]) + result = SimpleNamespace(predicted=response, metadata={"score": 0.0}) + figure = text._localization_card( + Image.new("RGB", (30, 30)), sample, result, task, "test", config, None + ) + try: + rail = figure.axes[1] + assert not rail.child_axes + assert np.array_equal( + rail.images[0].get_array(), np.asarray(structured._message_panel(message)) + ) + assert "BOX + TEXT F1" in [t.get_text() for t in rail.texts] + assert "0.0%" in [t.get_text() for t in rail.texts] + finally: + plt.close(figure)