From 93ea9d6a72c5f38b67bfc9188a71f1250716027f Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Mon, 27 Jul 2026 22:32:54 +0100 Subject: [PATCH 1/8] feat(lattice): add invalidatable autoconfig compiler --- apps/predbat/lattice_autoconfig.py | 868 ++++++++++++++++++ apps/predbat/tests/test_lattice_autoconfig.py | 492 ++++++++++ 2 files changed, 1360 insertions(+) create mode 100644 apps/predbat/lattice_autoconfig.py create mode 100644 apps/predbat/tests/test_lattice_autoconfig.py diff --git a/apps/predbat/lattice_autoconfig.py b/apps/predbat/lattice_autoconfig.py new file mode 100644 index 000000000..66922793e --- /dev/null +++ b/apps/predbat/lattice_autoconfig.py @@ -0,0 +1,868 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System - Lattice auto-configuration compiler +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +"""Pure Lattice fragment compilation and invalidation state machine. + +This module deliberately has no component registry, MQTT, Home Assistant, or +configuration-write dependency. Integrations expose snapshot readers and +invalidate their monotonically increasing generations. A later, separately +gated runtime component can supply a materializer. +""" + +# cspell:ignore autoconfig + +import copy +import hashlib +import json +import threading +from collections import deque +from dataclasses import dataclass +from enum import Enum +from types import MappingProxyType +from typing import Mapping, Optional + +from lattice_topology import TopologyValidationError, decode_topology, merge_topologies + + +class ProviderHealth(Enum): + """Health reported with a provider's fragment snapshot.""" + + HEALTHY = "healthy" + DEGRADED = "degraded" + OFFLINE = "offline" + + +class AliasRole(Enum): + """Semantic role assigned to a provider-qualified node alias.""" + + REFERENCE = "reference" + PRIMARY = "primary" + CONTROL = "control" + + +class CompileStatus(Enum): + """Observable compiler state after a drain.""" + + IDLE = "idle" + FRESH = "fresh" + DEGRADED = "degraded" + STALE = "stale" + + +class AutoConfigCompileError(ValueError): + """Raised internally when a candidate plan cannot be compiled safely.""" + + +@dataclass(frozen=True) +class ProviderAlias: + """A provider-local alias for one topology node.""" + + name: str + node_id: str + roles: frozenset = frozenset((AliasRole.REFERENCE,)) + + def __post_init__(self): + """Validate and normalise alias identity and roles.""" + if not isinstance(self.name, str) or not self.name.strip(): + raise ValueError("alias name must be a non-empty string") + if not isinstance(self.node_id, str) or not self.node_id.strip(): + raise ValueError("alias node_id must be a non-empty string") + roles = frozenset(self.roles) + if not roles or any(not isinstance(role, AliasRole) for role in roles): + raise ValueError("alias roles must contain AliasRole values") + object.__setattr__(self, "name", self.name.strip()) + object.__setattr__(self, "node_id", self.node_id.strip()) + object.__setattr__(self, "roles", roles) + + +@dataclass(frozen=True) +class ProviderIdentityAlias: + """A provider assertion that correlates a local node to a stable identity.""" + + kind: str + value: str + node_id: str + + def __post_init__(self): + """Validate and normalise the stable identity assertion.""" + if not isinstance(self.kind, str) or not self.kind.strip(): + raise ValueError("identity alias kind must be a non-empty string") + if not isinstance(self.value, str) or not self.value.strip(): + raise ValueError("identity alias value must be a non-empty string") + if not isinstance(self.node_id, str) or not self.node_id.strip(): + raise ValueError("identity alias node_id must be a non-empty string") + object.__setattr__(self, "kind", self.kind.strip().lower()) + object.__setattr__(self, "value", self.value.strip()) + object.__setattr__(self, "node_id", self.node_id.strip()) + + +@dataclass(frozen=True) +class ProviderSnapshot: + """One integration's immutable generation of health, topology, and aliases.""" + + provider_id: str + generation: int + health: ProviderHealth + topology_fragment: Mapping + aliases: tuple = () + identity_aliases: tuple = () + + def __post_init__(self): + """Validate scalar fields and detach caller-owned mutable data.""" + if not isinstance(self.provider_id, str) or not self.provider_id.strip(): + raise ValueError("provider_id must be a non-empty string") + if not isinstance(self.generation, int) or isinstance(self.generation, bool) or self.generation < 0: + raise ValueError("generation must be a non-negative integer") + if not isinstance(self.health, ProviderHealth): + raise ValueError("health must be a ProviderHealth") + if not isinstance(self.topology_fragment, Mapping): + raise ValueError("topology_fragment must be a mapping") + aliases = tuple(self.aliases) + if any(not isinstance(alias, ProviderAlias) for alias in aliases): + raise ValueError("aliases must contain ProviderAlias values") + identity_aliases = tuple(self.identity_aliases) + if any(not isinstance(alias, ProviderIdentityAlias) for alias in identity_aliases): + raise ValueError("identity_aliases must contain ProviderIdentityAlias values") + object.__setattr__(self, "provider_id", self.provider_id.strip()) + object.__setattr__(self, "topology_fragment", _freeze(copy.deepcopy(dict(self.topology_fragment)))) + object.__setattr__(self, "aliases", aliases) + object.__setattr__(self, "identity_aliases", identity_aliases) + + +@dataclass(frozen=True) +class AliasBinding: + """A compiled provider-qualified alias.""" + + qualified_name: str + provider_id: str + generation: int + node_id: str + roles: tuple + + +@dataclass(frozen=True) +class IdentityBinding: + """One qualified stable-identity assertion in the compiled plan.""" + + provider_id: str + generation: int + kind: str + value: str + local_node_id: str + canonical_node_id: str + + +@dataclass(frozen=True) +class FieldProvenance: + """Source coordinates for one generated plan field.""" + + field_path: str + provider_id: str + generation: int + source_path: str + + +@dataclass(frozen=True) +class AutoConfigField: + """One generated auto-configuration field and its provenance.""" + + name: str + value: object + provenance: tuple + + +@dataclass(frozen=True) +class AutoConfigPlan: + """Deterministic immutable result of compiling all usable fragments.""" + + digest: str + topology: Mapping + aliases: tuple + identity_aliases: tuple + fields: tuple + provenance: tuple + provider_generations: tuple + warnings: tuple + + +@dataclass(frozen=True) +class CompileIssue: + """A provider-local or global diagnostic from one compile attempt.""" + + code: str + detail: str + provider_id: Optional[str] = None + + +@dataclass(frozen=True) +class Invalidation: + """One accepted provider cause retained across compile coalescing.""" + + provider_id: str + generation: int + reason: str + + +@dataclass(frozen=True) +class MaterializationRequest: + """A pure hand-off to a future materializer.""" + + plan: AutoConfigPlan + feedback_token: str + + +@dataclass(frozen=True) +class CompileRun: + """Summary returned after draining one compile and one optional follow-up.""" + + attempts: int + status: CompileStatus + plan: Optional[AutoConfigPlan] + issues: tuple + invalidations: tuple + materializations: int + pending: bool + + +def _freeze(value): + """Recursively detach and freeze JSON-like values.""" + if isinstance(value, Mapping): + return MappingProxyType({str(key): _freeze(item) for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))}) + if isinstance(value, (list, tuple)): + return tuple(_freeze(item) for item in value) + if isinstance(value, (str, int, float, bool)) or value is None: + return value + raise ValueError("unsupported non-JSON value {!r}".format(type(value).__name__)) + + +def _plain(value): + """Convert frozen JSON-like values and enums to canonical plain data.""" + if isinstance(value, Mapping): + return {str(key): _plain(item) for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))} + if isinstance(value, (list, tuple, frozenset)): + items = [_plain(item) for item in value] + return sorted(items) if isinstance(value, frozenset) else items + if isinstance(value, Enum): + return value.value + if isinstance(value, (str, int, float, bool)) or value is None: + return value + raise ValueError("unsupported canonical value {!r}".format(type(value).__name__)) + + +def _canonical_json(value): + """Encode a value using the compiler's deterministic canonical form.""" + return json.dumps(_plain(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False) + + +def _semantic_topology(site): + """Remove producer bookkeeping that cannot change materialized config.""" + topology = _plain(site) + topology.pop("docVersion", None) + topology.pop("producer", None) + return topology + + +def _fingerprint_snapshot(snapshot): + """Bind a provider generation to exactly one health/fragment/alias value.""" + aliases = [ + { + "name": alias.name, + "node_id": alias.node_id, + "roles": sorted(role.value for role in alias.roles), + } + for alias in sorted(snapshot.aliases, key=lambda item: (item.name, item.node_id, tuple(sorted(role.value for role in item.roles)))) + ] + identity_aliases = [ + { + "kind": alias.kind, + "value": alias.value, + "node_id": alias.node_id, + } + for alias in sorted(snapshot.identity_aliases, key=lambda item: (item.kind, item.value, item.node_id)) + ] + payload = { + "provider": snapshot.provider_id, + "generation": snapshot.generation, + "health": snapshot.health.value, + "fragment": snapshot.topology_fragment, + "aliases": aliases, + "identity_aliases": identity_aliases, + } + return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() + + +def _node_identity(node): + """Return safety-relevant identity fields for collision detection.""" + attributes = node.get("attributes") if isinstance(node.get("attributes"), Mapping) else {} + return { + "kind": node.get("kind"), + "deviceType": node.get("deviceType"), + "serial": attributes.get("serial"), + "manufacturer": attributes.get("manufacturer"), + "model": attributes.get("model"), + } + + +def _identity_conflicts(left, right): + """Return conflicting non-empty identity fields.""" + return tuple(sorted(key for key in left if left.get(key) not in (None, "") and right.get(key) not in (None, "") and left[key] != right[key])) + + +def _document_node_ids(document, provider_id): + """Validate node identities within a fragment and return their ids.""" + nodes = document.get("nodes", ()) + seen = {} + for node in nodes: + node_id = str(node["id"]) + if node_id in seen: + raise AutoConfigCompileError("provider {} repeats node identity {}".format(provider_id, node_id)) + seen[node_id] = _node_identity(node) + return seen + + +def _find_group(parent, node): + """Return a union-find root with path compression.""" + root = node + while parent[root] != root: + root = parent[root] + while parent[node] != node: + next_node = parent[node] + parent[node] = root + node = next_node + return root + + +def _join_groups(parent, left, right): + """Deterministically join two union-find identity groups.""" + left_root = _find_group(parent, left) + right_root = _find_group(parent, right) + if left_root == right_root: + return + if left_root < right_root: + parent[right_root] = left_root + else: + parent[left_root] = right_root + + +def _correlate_identities(snapshots, documents, provider_nodes): + """Correlate provider-local node ids through qualified stable assertions.""" + parent = {} + for snapshot in snapshots: + for node_id in provider_nodes[snapshot.provider_id]: + node_key = (snapshot.provider_id, node_id) + parent[node_key] = node_key + + assertions = {} + identity_owners = {} + for snapshot in snapshots: + provider_assertions = set() + for alias in sorted(snapshot.identity_aliases, key=lambda item: (item.kind, item.value, item.node_id)): + node_key = (snapshot.provider_id, alias.node_id) + if node_key not in parent: + raise AutoConfigCompileError("identity alias {}:{}:{} targets unknown provider-local node {}".format(snapshot.provider_id, alias.kind, alias.value, alias.node_id)) + assertion_key = (snapshot.provider_id, alias.kind, alias.value) + if assertion_key in provider_assertions: + raise AutoConfigCompileError("identity alias collision for {}:{}:{}".format(*assertion_key)) + provider_assertions.add(assertion_key) + identity_key = (alias.kind, alias.value) + previous = identity_owners.get(identity_key) + if previous is not None: + _join_groups(parent, previous, node_key) + else: + identity_owners[identity_key] = node_key + assertions[(snapshot.provider_id, alias.kind, alias.value, alias.node_id)] = node_key + + groups = {} + for node_key in parent: + groups.setdefault(_find_group(parent, node_key), []).append(node_key) + group_assertions = {} + for assertion_key, node_key in assertions.items(): + group_assertions.setdefault(_find_group(parent, node_key), []).append(assertion_key) + + canonical_by_node = {} + canonical_owners = {} + for root, members in sorted(groups.items()): + providers = [member[0] for member in members] + if len(providers) != len(set(providers)): + raise AutoConfigCompileError("one provider correlates multiple local nodes in identity group {}".format(sorted(members))) + + aliases_by_kind = {} + for _provider_id, kind, value, _node_id in group_assertions.get(root, ()): + aliases_by_kind.setdefault(kind, set()).add(value) + conflicts = {kind: values for kind, values in aliases_by_kind.items() if len(values) > 1} + if conflicts: + detail = ", ".join("{}={}".format(kind, sorted(values)) for kind, values in sorted(conflicts.items())) + raise AutoConfigCompileError("strong identity aliases conflict ({})".format(detail)) + + identities = [provider_nodes[provider_id][node_id] for provider_id, node_id in members] + for index, left in enumerate(identities): + for right in identities[index + 1 :]: + identity_conflicts = _identity_conflicts(left, right) + if identity_conflicts: + raise AutoConfigCompileError("identity collision in correlated group {} ({})".format(sorted(members), ", ".join(identity_conflicts))) + + stable_aliases = sorted((kind, next(iter(values))) for kind, values in aliases_by_kind.items()) + if stable_aliases: + kind, value = min(stable_aliases, key=lambda item: (item[0] != "serial", item[0], item[1])) + canonical_node_id = "identity:{}:{}".format(kind, value) + else: + provider_id, node_id = members[0] + canonical_node_id = "provider:{}:{}".format(provider_id, node_id) + previous_group = canonical_owners.get(canonical_node_id) + if previous_group is not None and previous_group != root: + raise AutoConfigCompileError("canonical identity collision for {}".format(canonical_node_id)) + canonical_owners[canonical_node_id] = root + for member in members: + canonical_by_node[member] = canonical_node_id + + normalized_documents = [] + for snapshot, document in zip(snapshots, documents): + normalized = copy.deepcopy(document) + local_map = {node_id: canonical_by_node[(snapshot.provider_id, node_id)] for node_id in provider_nodes[snapshot.provider_id]} + for node in normalized.get("nodes", ()): + node["id"] = local_map[str(node["id"])] + for relationship in normalized.get("relationships", ()): + for endpoint in ("from", "to"): + endpoint_id = str(relationship.get(endpoint)) + if endpoint_id in local_map: + relationship[endpoint] = local_map[endpoint_id] + normalized_documents.append(normalized) + + identity_bindings = tuple( + sorted( + ( + IdentityBinding( + provider_id=provider_id, + generation=next(snapshot.generation for snapshot in snapshots if snapshot.provider_id == provider_id), + kind=kind, + value=value, + local_node_id=node_id, + canonical_node_id=canonical_by_node[(provider_id, node_id)], + ) + for provider_id, kind, value, node_id in assertions + ), + key=lambda item: (item.provider_id, item.kind, item.value, item.local_node_id), + ) + ) + return tuple(normalized_documents), canonical_by_node, identity_bindings + + +def _field_provenance(snapshots, bindings, identity_bindings, primary_target, control_target, topology_snapshot, canonical_by_node): + """Build deterministic source coordinates for every generated field.""" + provenance = [] + for snapshot in snapshots: + provenance.append(FieldProvenance("/topology", snapshot.provider_id, snapshot.generation, "/")) + for binding in bindings: + provenance.append( + FieldProvenance( + "/aliases/{}/node_id".format(binding.qualified_name), + binding.provider_id, + binding.generation, + "/aliases/{}".format(binding.qualified_name.split(":", 1)[1]), + ) + ) + for binding in identity_bindings: + provenance.append( + FieldProvenance( + "/identity_aliases/{}:{}:{}/canonical_node_id".format(binding.provider_id, binding.kind, binding.value), + binding.provider_id, + binding.generation, + "/identity_aliases/{}:{}".format(binding.kind, binding.value), + ) + ) + for field_path, target, role in (("/primary_target", primary_target, AliasRole.PRIMARY), ("/control_target", control_target, AliasRole.CONTROL)): + if target is None: + continue + for binding in bindings: + if binding.node_id == target and role.value in binding.roles: + provenance.append( + FieldProvenance( + field_path, + binding.provider_id, + binding.generation, + "/aliases/{}".format(binding.qualified_name.split(":", 1)[1]), + ) + ) + generations = {snapshot.provider_id: snapshot.generation for snapshot in snapshots} + local_by_canonical = {(provider_id, canonical_node_id): local_node_id for (provider_id, local_node_id), canonical_node_id in canonical_by_node.items()} + for key, source in topology_snapshot.provenance.items(): + local_node_id = local_by_canonical[(source.provider, source.node_id)] + provenance.append( + FieldProvenance( + "/topology/nodes/{}/capabilities/{}@{}".format(key[0], key[1], key[2]), + source.provider, + generations[source.provider], + "/nodes/{}/capabilities/{}@{}".format(local_node_id, source.capability, source.access_path), + ) + ) + return tuple(sorted(provenance, key=lambda item: (item.field_path, item.provider_id, item.generation, item.source_path))) + + +def compile_auto_config(snapshots): + """Compile usable provider snapshots into one deterministic immutable plan.""" + snapshots = tuple(sorted(snapshots, key=lambda item: item.provider_id)) + if not snapshots: + raise AutoConfigCompileError("no usable provider snapshots") + provider_ids = [snapshot.provider_id for snapshot in snapshots] + if len(provider_ids) != len(set(provider_ids)): + raise AutoConfigCompileError("duplicate provider snapshots are not allowed") + + documents = [] + provider_nodes = {} + for snapshot in snapshots: + document = decode_topology(_plain(snapshot.topology_fragment)) + if document["scope"] != "fragment": + raise AutoConfigCompileError("provider {} topology scope must be fragment".format(snapshot.provider_id)) + document_provider = document["producer"]["provider"] + if document_provider != snapshot.provider_id: + raise AutoConfigCompileError("provider {} supplied fragment owned by {}".format(snapshot.provider_id, document_provider)) + nodes = _document_node_ids(document, snapshot.provider_id) + provider_nodes[snapshot.provider_id] = nodes + documents.append(document) + documents, canonical_by_node, identity_bindings = _correlate_identities(snapshots, documents, provider_nodes) + + bindings = [] + qualified = set() + for snapshot in snapshots: + for alias in sorted(snapshot.aliases, key=lambda item: (item.name, item.node_id, tuple(sorted(role.value for role in item.roles)))): + qualified_name = "{}:{}".format(snapshot.provider_id, alias.name) + if qualified_name in qualified: + raise AutoConfigCompileError("alias collision for {}".format(qualified_name)) + qualified.add(qualified_name) + if alias.node_id not in provider_nodes[snapshot.provider_id]: + raise AutoConfigCompileError("alias {} targets unknown provider-local node {}".format(qualified_name, alias.node_id)) + bindings.append( + AliasBinding( + qualified_name=qualified_name, + provider_id=snapshot.provider_id, + generation=snapshot.generation, + node_id=canonical_by_node[(snapshot.provider_id, alias.node_id)], + roles=tuple(sorted(role.value for role in alias.roles)), + ) + ) + bindings = tuple(sorted(bindings, key=lambda item: item.qualified_name)) + + targets = {} + for role in (AliasRole.PRIMARY, AliasRole.CONTROL): + role_targets = sorted({binding.node_id for binding in bindings if role.value in binding.roles}) + if len(role_targets) > 1: + raise AutoConfigCompileError("ambiguous {} target: {}".format(role.value, ", ".join(role_targets))) + targets[role] = role_targets[0] if role_targets else None + + topology_snapshot = merge_topologies(documents) + fields = [] + for binding in bindings: + source = FieldProvenance( + "/aliases/{}/node_id".format(binding.qualified_name), + binding.provider_id, + binding.generation, + "/aliases/{}".format(binding.qualified_name.split(":", 1)[1]), + ) + fields.append(AutoConfigField("alias.{}".format(binding.qualified_name), binding.node_id, (source,))) + for name, role in (("primary_target", AliasRole.PRIMARY), ("control_target", AliasRole.CONTROL)): + target = targets[role] + if target is None: + continue + sources = tuple( + FieldProvenance( + "/{}".format(name), + binding.provider_id, + binding.generation, + "/aliases/{}".format(binding.qualified_name.split(":", 1)[1]), + ) + for binding in bindings + if binding.node_id == target and role.value in binding.roles + ) + fields.append(AutoConfigField(name, target, sources)) + fields = tuple(sorted(fields, key=lambda item: item.name)) + + semantic = { + "topology": _semantic_topology(topology_snapshot.site), + "aliases": [ + { + "qualified_name": binding.qualified_name, + "node_id": binding.node_id, + "roles": binding.roles, + } + for binding in bindings + ], + "identity_aliases": [ + { + "provider_id": binding.provider_id, + "kind": binding.kind, + "value": binding.value, + "local_node_id": binding.local_node_id, + "canonical_node_id": binding.canonical_node_id, + } + for binding in identity_bindings + ], + "fields": [{"name": field.name, "value": field.value} for field in fields], + } + digest = hashlib.sha256(_canonical_json(semantic).encode("utf-8")).hexdigest() + provenance = _field_provenance( + snapshots, + bindings, + identity_bindings, + targets[AliasRole.PRIMARY], + targets[AliasRole.CONTROL], + topology_snapshot, + canonical_by_node, + ) + return AutoConfigPlan( + digest=digest, + topology=_freeze(topology_snapshot.site), + aliases=bindings, + identity_aliases=identity_bindings, + fields=fields, + provenance=provenance, + provider_generations=tuple((snapshot.provider_id, snapshot.generation) for snapshot in snapshots), + warnings=tuple(topology_snapshot.warnings), + ) + + +class LatticeAutoConfigCompiler: + """Thread-safe invalidation, compilation, and last-known-good coordinator.""" + + def __init__(self, readers=None, materializer=None): + """Create an idle compiler over provider snapshot readers.""" + self._readers = {} + self._materializer = materializer + self._lock = threading.RLock() + self._compiling = False + self._pending = False + self._follow_up = False + self._requested_generations = {} + self._observed_generations = {} + self._generation_fingerprints = {} + self._invalidations = set() + self._feedback_tokens = deque(maxlen=64) + self._token_counter = 0 + self._active_plan = None + self._status = CompileStatus.IDLE + self._issues = () + for provider_id, reader in (readers or {}).items(): + self.register_provider(provider_id, reader) + + @property + def active_plan(self): + """Return the last-known-good immutable plan.""" + with self._lock: + return self._active_plan + + @property + def status(self): + """Return the current observable compile status.""" + with self._lock: + return self._status + + def register_provider(self, provider_id, reader): + """Register one integration's side-effect-free snapshot reader.""" + if not isinstance(provider_id, str) or not provider_id.strip(): + raise ValueError("provider_id must be a non-empty string") + if not callable(reader): + raise ValueError("provider reader must be callable") + provider_id = provider_id.strip() + with self._lock: + if self._compiling: + raise RuntimeError("cannot register a provider during compilation") + if provider_id in self._readers: + raise ValueError("provider {} is already registered".format(provider_id)) + self._readers[provider_id] = reader + self._pending = True + + def invalidate(self, provider_id, generation, reason, feedback_token=None): + """Accept a newer provider generation and schedule recompilation. + + Returns ``False`` for a stale/replayed generation or a materializer + feedback event. Unknown providers fail closed instead of silently + creating an unobservable input. + """ + if not isinstance(generation, int) or isinstance(generation, bool) or generation < 0: + raise ValueError("generation must be a non-negative integer") + if not isinstance(reason, str) or not reason.strip(): + raise ValueError("reason must be a non-empty string") + with self._lock: + if provider_id not in self._readers: + raise KeyError("unknown provider {}".format(provider_id)) + if feedback_token is not None and feedback_token in self._feedback_tokens: + return False + previous = max(self._requested_generations.get(provider_id, -1), self._observed_generations.get(provider_id, -1)) + if generation <= previous: + return False + self._requested_generations[provider_id] = generation + self._invalidations.add(Invalidation(provider_id, generation, reason.strip())) + if self._compiling: + self._follow_up = True + else: + self._pending = True + return True + + def _read_all(self): + """Fresh-read every registered provider, isolating local failures.""" + with self._lock: + readers = tuple(sorted(self._readers.items())) + usable = [] + issues = [] + for provider_id, reader in readers: + try: + snapshot = reader() + if not isinstance(snapshot, ProviderSnapshot): + raise ValueError("reader must return ProviderSnapshot") + if snapshot.provider_id != provider_id: + raise ValueError("reader returned provider {}".format(snapshot.provider_id)) + fingerprint = _fingerprint_snapshot(snapshot) + with self._lock: + required_generation = self._requested_generations.get(provider_id, -1) + previous_generation = self._observed_generations.get(provider_id, -1) + previous_fingerprint = self._generation_fingerprints.get((provider_id, snapshot.generation)) + if snapshot.generation < previous_generation: + raise ValueError("snapshot generation {} regressed from {}".format(snapshot.generation, previous_generation)) + if previous_fingerprint is not None and previous_fingerprint != fingerprint: + raise ValueError("snapshot generation {} was reused with different content".format(snapshot.generation)) + if snapshot.generation < required_generation: + raise ValueError("snapshot generation {} is behind invalidation {}".format(snapshot.generation, required_generation)) + self._observed_generations[provider_id] = snapshot.generation + self._requested_generations[provider_id] = max(required_generation, snapshot.generation) + self._generation_fingerprints[(provider_id, snapshot.generation)] = fingerprint + if snapshot.health is ProviderHealth.OFFLINE: + issues.append(CompileIssue("provider_offline", "provider is offline", provider_id)) + continue + decode_topology(_plain(snapshot.topology_fragment)) + if snapshot.health is ProviderHealth.DEGRADED: + issues.append(CompileIssue("provider_degraded", "provider reports degraded health", provider_id)) + usable.append(snapshot) + except (AutoConfigCompileError, TopologyValidationError, TypeError, ValueError) as exc: + issues.append(CompileIssue("provider_invalid", str(exc), provider_id)) + except Exception as exc: + issues.append(CompileIssue("provider_read_failed", "{}: {}".format(type(exc).__name__, exc), provider_id)) + return tuple(usable), tuple(issues) + + def _compile_attempt(self): + """Compile one fresh all-provider snapshot read.""" + snapshots, issues = self._read_all() + with self._lock: + active_providers = set(dict(self._active_plan.provider_generations)) if self._active_plan is not None else set() + usable_providers = {snapshot.provider_id for snapshot in snapshots} + unavailable_active = sorted(active_providers - usable_providers) + if unavailable_active: + detail = "previously active provider(s) unavailable: {}".format(", ".join(unavailable_active)) + return None, issues + (CompileIssue("active_provider_unavailable", detail), CompileIssue("compile_failed", detail)) + try: + plan = compile_auto_config(snapshots) + except (AutoConfigCompileError, TopologyValidationError, TypeError, ValueError) as exc: + return None, issues + (CompileIssue("compile_failed", str(exc)),) + return plan, issues + + def _materialize_if_changed(self, plan): + """Hand a changed plan to the injected materializer exactly once.""" + with self._lock: + if self._materializer is None or (self._active_plan is not None and plan.digest == self._active_plan.digest): + return 0, () + self._token_counter += 1 + feedback_token = "lattice-autoconfig-{}".format(self._token_counter) + self._feedback_tokens.append(feedback_token) + try: + self._materializer(MaterializationRequest(plan, feedback_token)) + except Exception as exc: + return 0, (CompileIssue("materialization_failed", "{}: {}".format(type(exc).__name__, exc)),) + return 1, () + + def drain(self): + """Run one pending compile plus at most one guaranteed follow-up.""" + with self._lock: + if self._compiling or not self._pending: + return CompileRun(0, self._status, self._active_plan, self._issues, (), 0, self._pending) + self._compiling = True + self._pending = False + self._follow_up = False + invalidations = tuple(sorted(self._invalidations, key=lambda item: (item.provider_id, item.generation, item.reason))) + self._invalidations.clear() + + attempts = 0 + materializations = 0 + aggregate_issues = () + final_issues = () + candidate = None + try: + while attempts < 2: + attempts += 1 + plan, issues = self._compile_attempt() + aggregate_issues += issues + + with self._lock: + follow_up = self._follow_up + self._follow_up = False + if follow_up: + invalidations += tuple(sorted(self._invalidations, key=lambda item: (item.provider_id, item.generation, item.reason))) + self._invalidations.clear() + if follow_up and attempts < 2: + continue + if follow_up: + self._pending = True + superseded = CompileIssue("compile_superseded", "candidate superseded by an invalidation during the bounded follow-up") + aggregate_issues += (superseded,) + final_issues = issues + (superseded,) + candidate = None + else: + final_issues = issues + candidate = plan + break + + with self._lock: + # Linearize the final candidate decision and materializer call + # with invalidation. RLock permits same-thread token feedback. + if self._follow_up: + self._pending = True + invalidations += tuple(sorted(self._invalidations, key=lambda item: (item.provider_id, item.generation, item.reason))) + self._invalidations.clear() + self._follow_up = False + superseded = CompileIssue("compile_superseded", "candidate superseded before materialization") + aggregate_issues += (superseded,) + final_issues += (superseded,) + candidate = None + if candidate is not None: + count, materializer_issues = self._materialize_if_changed(candidate) + materializations += count + aggregate_issues += materializer_issues + final_issues += materializer_issues + if not materializer_issues: + self._active_plan = candidate + if self._follow_up: + self._pending = True + invalidations += tuple(sorted(self._invalidations, key=lambda item: (item.provider_id, item.generation, item.reason))) + self._invalidations.clear() + self._follow_up = False + + fatal_codes = ("compile_failed", "compile_superseded", "materialization_failed") + fatal = any(issue.code in fatal_codes for issue in final_issues) + if fatal: + # Retry is caller/backoff-driven. It deliberately does not + # consume another attempt inside this drain. + self._pending = True + if self._active_plan is None or fatal: + self._status = CompileStatus.STALE + elif final_issues: + self._status = CompileStatus.DEGRADED + else: + self._status = CompileStatus.FRESH + self._issues = aggregate_issues + self._compiling = False + result = CompileRun( + attempts, + self._status, + self._active_plan, + aggregate_issues, + tuple(sorted(set(invalidations), key=lambda item: (item.provider_id, item.generation, item.reason))), + materializations, + self._pending, + ) + return result + except Exception: + with self._lock: + self._compiling = False + if self._follow_up: + self._pending = True + self._follow_up = False + raise diff --git a/apps/predbat/tests/test_lattice_autoconfig.py b/apps/predbat/tests/test_lattice_autoconfig.py new file mode 100644 index 000000000..5a1d14aab --- /dev/null +++ b/apps/predbat/tests/test_lattice_autoconfig.py @@ -0,0 +1,492 @@ +"""Tests for pure Lattice fragment auto-configuration compilation.""" + +# cspell:ignore autoconfig + +import os +import sys +import threading +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from lattice_autoconfig import ( + AliasRole, + AutoConfigCompileError, + CompileStatus, + LatticeAutoConfigCompiler, + ProviderAlias, + ProviderHealth, + ProviderIdentityAlias, + ProviderSnapshot, + compile_auto_config, +) + + +def fragment(provider, generation, node_id="INV1", kind="inverter"): + """Build a compact provider-owned topology fragment.""" + access_path = "{}-path".format(provider) + return { + "topologyVersion": "0.3.0", + "scope": "fragment", + "docVersion": generation, + "producer": {"name": provider, "provider": provider, "authority": 10}, + "nodes": [ + { + "id": node_id, + "kind": kind, + "deviceType": "hybrid", + "accessPaths": [{"id": access_path, "provider": provider, "preference": 10}], + "capabilities": [ + { + "capability": "battery.target_soc", + "accessPath": access_path, + "ref": 1, + "shape": "setpoint", + "control": {"protocol": "mqtt"}, + } + ], + } + ], + } + + +def snapshot(provider, generation=1, node_id="INV1", kind="inverter", health=ProviderHealth.HEALTHY, aliases=(), identity_aliases=()): + """Build one typed provider snapshot.""" + return ProviderSnapshot(provider, generation, health, fragment(provider, generation, node_id=node_id, kind=kind), aliases, identity_aliases) + + +class MutableReader: + """Thread-safe-enough mutable snapshot reader for deterministic tests.""" + + def __init__(self, value): + """Store the first snapshot and initialise the call count.""" + self.value = value + self.calls = 0 + + def __call__(self): + """Return the current snapshot and count the fresh read.""" + self.calls += 1 + return self.value + + +class TestPlanCompilation(unittest.TestCase): + """Compiler output is safe, immutable, deterministic, and attributable.""" + + def test_order_independent_digest_and_provider_qualified_aliases(self): + """Input order and shared local alias names cannot alter a plan.""" + alias = ProviderAlias("battery", "INV1", frozenset((AliasRole.REFERENCE, AliasRole.PRIMARY, AliasRole.CONTROL))) + gateway = snapshot("gateway", aliases=(alias,), identity_aliases=(ProviderIdentityAlias("serial", "SER123", "INV1"),)) + cloud = snapshot("cloud", aliases=(alias,), identity_aliases=(ProviderIdentityAlias("serial", "SER123", "INV1"),)) + + left = compile_auto_config((gateway, cloud)) + right = compile_auto_config((cloud, gateway)) + + self.assertEqual(left.digest, right.digest) + self.assertEqual([binding.qualified_name for binding in left.aliases], ["cloud:battery", "gateway:battery"]) + self.assertEqual(dict(left.provider_generations), {"cloud": 1, "gateway": 1}) + self.assertEqual({field.name for field in left.fields}, {"alias.cloud:battery", "alias.gateway:battery", "control_target", "primary_target"}) + self.assertTrue(all(field.provenance for field in left.fields)) + with self.assertRaises(TypeError): + left.topology["scope"] = "fragment" + + def test_generation_bookkeeping_does_not_change_semantic_digest(self): + """An identical newer fragment generation avoids materialization churn.""" + alias = ProviderAlias("battery", "INV1", frozenset((AliasRole.PRIMARY,))) + first = compile_auto_config((snapshot("gateway", generation=1, aliases=(alias,)),)) + second = compile_auto_config((snapshot("gateway", generation=2, aliases=(alias,)),)) + + self.assertEqual(first.digest, second.digest) + self.assertNotEqual(first.provider_generations, second.provider_generations) + + def test_shared_stable_identity_correlates_different_local_nodes(self): + """Gateway and cloud assertions for one serial compile to one node.""" + gateway_alias = ProviderIdentityAlias("serial", "SER123", "gw-local-1") + cloud_alias = ProviderIdentityAlias("serial", "SER123", "cloud-local-9") + primary = ProviderAlias("battery", "gw-local-1", frozenset((AliasRole.PRIMARY, AliasRole.CONTROL))) + gateway = snapshot("gateway", node_id="gw-local-1", aliases=(primary,), identity_aliases=(gateway_alias,)) + cloud = snapshot("cloud", node_id="cloud-local-9", identity_aliases=(cloud_alias,)) + + plan = compile_auto_config((gateway, cloud)) + + self.assertEqual(len(plan.topology["nodes"]), 1) + self.assertEqual(plan.topology["nodes"][0]["id"], "identity:serial:SER123") + self.assertEqual({path["id"] for path in plan.topology["nodes"][0]["accessPaths"]}, {"gateway-path", "cloud-path"}) + self.assertEqual(plan.aliases[0].node_id, "identity:serial:SER123") + self.assertEqual({binding.provider_id for binding in plan.identity_aliases}, {"gateway", "cloud"}) + capability_sources = {item.provider_id: item.source_path for item in plan.provenance if "/capabilities/" in item.field_path} + self.assertIn("gw-local-1", capability_sources["gateway"]) + self.assertIn("cloud-local-9", capability_sources["cloud"]) + + def test_equal_provider_local_ids_do_not_implicitly_correlate(self): + """Matching local labels remain separate without a shared stable identity.""" + plan = compile_auto_config((snapshot("gateway", node_id="INV1"), snapshot("cloud", node_id="INV1"))) + + self.assertEqual( + {node["id"] for node in plan.topology["nodes"]}, + {"provider:gateway:INV1", "provider:cloud:INV1"}, + ) + + def test_duplicate_provider_snapshots_fail_closed(self): + """The pure compile entry point cannot accept two generations of one provider.""" + with self.assertRaisesRegex(AutoConfigCompileError, "duplicate provider"): + compile_auto_config((snapshot("gateway", generation=1), snapshot("gateway", generation=2))) + + def test_provider_cannot_reuse_stable_identity_for_two_nodes(self): + """One integration cannot correlate two local nodes to one identity.""" + document = fragment("gateway", 1, node_id="INV1") + second = dict(document["nodes"][0]) + second["id"] = "INV2" + document["nodes"].append(second) + aliases = (ProviderIdentityAlias("serial", "SER123", "INV1"), ProviderIdentityAlias("serial", "SER123", "INV2")) + provider = ProviderSnapshot("gateway", 1, ProviderHealth.HEALTHY, document, (), aliases) + + with self.assertRaisesRegex(AutoConfigCompileError, "identity alias collision"): + compile_auto_config((provider,)) + + def test_correlated_strong_identity_values_cannot_conflict(self): + """A cross-kind correlation cannot conceal conflicting serial values.""" + gateway_aliases = ( + ProviderIdentityAlias("serial", "SER-A", "gw"), + ProviderIdentityAlias("mac", "AA:BB", "gw"), + ) + cloud_aliases = ( + ProviderIdentityAlias("serial", "SER-B", "cloud"), + ProviderIdentityAlias("mac", "AA:BB", "cloud"), + ) + gateway = snapshot("gateway", node_id="gw", identity_aliases=gateway_aliases) + cloud = snapshot("cloud", node_id="cloud", identity_aliases=cloud_aliases) + + with self.assertRaisesRegex(AutoConfigCompileError, "strong identity aliases conflict"): + compile_auto_config((gateway, cloud)) + + def test_duplicate_qualified_alias_fails_closed(self): + """One provider cannot publish the same qualified alias twice.""" + aliases = (ProviderAlias("battery", "INV1"), ProviderAlias("battery", "INV1")) + with self.assertRaisesRegex(AutoConfigCompileError, "alias collision"): + compile_auto_config((snapshot("gateway", aliases=aliases),)) + + def test_identity_collision_fails_closed(self): + """Conflicting identity fields for one node cannot be authority-merged.""" + gateway_alias = ProviderIdentityAlias("serial", "SER123", "INV1") + cloud_alias = ProviderIdentityAlias("serial", "SER123", "INV1") + with self.assertRaisesRegex(AutoConfigCompileError, "identity collision"): + compile_auto_config( + ( + snapshot("gateway", kind="inverter", identity_aliases=(gateway_alias,)), + snapshot("cloud", kind="battery", identity_aliases=(cloud_alias,)), + ) + ) + + def test_ambiguous_primary_and_control_targets_fail_closed(self): + """Several distinct target nodes cannot silently pick a winner.""" + primary_a = ProviderAlias("battery", "INV1", frozenset((AliasRole.PRIMARY,))) + primary_b = ProviderAlias("battery", "INV2", frozenset((AliasRole.PRIMARY,))) + with self.assertRaisesRegex(AutoConfigCompileError, "ambiguous primary"): + compile_auto_config((snapshot("gateway", node_id="INV1", aliases=(primary_a,)), snapshot("cloud", node_id="INV2", aliases=(primary_b,)))) + + control_a = ProviderAlias("control", "INV1", frozenset((AliasRole.CONTROL,))) + control_b = ProviderAlias("control", "INV2", frozenset((AliasRole.CONTROL,))) + with self.assertRaisesRegex(AutoConfigCompileError, "ambiguous control"): + compile_auto_config((snapshot("gateway", node_id="INV1", aliases=(control_a,)), snapshot("cloud", node_id="INV2", aliases=(control_b,)))) + + def test_alias_must_target_provider_local_identity(self): + """An alias cannot smuggle a target owned only by another provider.""" + bad = ProviderAlias("battery", "INV2") + with self.assertRaisesRegex(AutoConfigCompileError, "unknown provider-local node"): + compile_auto_config((snapshot("gateway", node_id="INV1", aliases=(bad,)), snapshot("cloud", node_id="INV2"))) + + +class TestInvalidationStateMachine(unittest.TestCase): + """Invalidations coalesce without losing freshness or last-known-good state.""" + + def test_replayed_and_stale_generations_are_rejected(self): + """A generation is accepted once and never rolls backwards.""" + reader = MutableReader(snapshot("gateway", generation=1)) + compiler = LatticeAutoConfigCompiler({"gateway": reader}) + + self.assertTrue(compiler.invalidate("gateway", 1, "first telemetry")) + self.assertFalse(compiler.invalidate("gateway", 1, "duplicate")) + self.assertFalse(compiler.invalidate("gateway", 0, "stale")) + run = compiler.drain() + self.assertEqual(run.attempts, 1) + self.assertFalse(compiler.invalidate("gateway", 1, "observed replay")) + + def test_burst_of_ten_simultaneous_invalidations_coalesces(self): + """Ten integrations invalidating together need only one all-provider read.""" + readers = {name: MutableReader(snapshot(name, generation=1, node_id="INV1")) for name in ("a", "b", "c", "d", "e", "f", "g", "h", "i", "j")} + compiler = LatticeAutoConfigCompiler(readers) + barrier = threading.Barrier(len(readers)) + accepted = [] + + def invalidate(name): + """Release the burst together and record acceptance.""" + barrier.wait() + accepted.append(compiler.invalidate(name, 1, "simultaneous discovery")) + + threads = [threading.Thread(target=invalidate, args=(name,)) for name in readers] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + run = compiler.drain() + self.assertTrue(all(accepted)) + self.assertEqual(run.attempts, 1) + self.assertLessEqual(run.attempts, 2) + self.assertTrue(all(reader.calls == 1 for reader in readers.values())) + + def test_invalidation_during_compile_guarantees_one_fresh_follow_up(self): + """A mid-compile generation change is included by one bounded follow-up.""" + entered = threading.Event() + release = threading.Event() + state = {"value": snapshot("gateway", generation=1), "calls": 0} + + def reader(): + """Block only the first read after capturing its generation.""" + state["calls"] += 1 + value = state["value"] + if state["calls"] == 1: + entered.set() + release.wait(5) + return value + + requests = [] + compiler = LatticeAutoConfigCompiler({"gateway": reader}, requests.append) + result = {} + worker = threading.Thread(target=lambda: result.setdefault("run", compiler.drain())) + worker.start() + self.assertTrue(entered.wait(5)) + state["value"] = snapshot("gateway", generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "new device")) + release.set() + worker.join(5) + + run = result["run"] + self.assertEqual(run.attempts, 2) + self.assertEqual(state["calls"], 2) + self.assertEqual(dict(run.plan.provider_generations), {"gateway": 2}) + self.assertEqual([dict(request.plan.provider_generations) for request in requests], [{"gateway": 2}]) + self.assertEqual(run.materializations, 1) + self.assertFalse(run.pending) + + def test_all_accepted_invalidation_causes_survive_coalescing(self): + """Execution coalesces while distinct provider causes remain auditable.""" + reader = MutableReader(snapshot("gateway", generation=2)) + compiler = LatticeAutoConfigCompiler({"gateway": reader}) + + self.assertTrue(compiler.invalidate("gateway", 1, "health changed")) + self.assertTrue(compiler.invalidate("gateway", 2, "topology changed")) + run = compiler.drain() + + self.assertEqual( + {(item.provider_id, item.generation, item.reason) for item in run.invalidations}, + { + ("gateway", 1, "health changed"), + ("gateway", 2, "topology changed"), + }, + ) + self.assertEqual(run.attempts, 1) + + def test_single_flight_rejects_a_concurrent_drain(self): + """Only one caller can own the compile flight.""" + entered = threading.Event() + release = threading.Event() + + def reader(): + """Hold the active flight until the competing drain returns.""" + entered.set() + release.wait(5) + return snapshot("gateway") + + compiler = LatticeAutoConfigCompiler({"gateway": reader}) + result = {} + worker = threading.Thread(target=lambda: result.setdefault("run", compiler.drain())) + worker.start() + self.assertTrue(entered.wait(5)) + concurrent = compiler.drain() + release.set() + worker.join(5) + + self.assertEqual(concurrent.attempts, 0) + self.assertEqual(result["run"].attempts, 1) + + def test_provider_failures_and_offline_health_are_isolated(self): + """Bad and offline producers do not erase a healthy provider's plan.""" + good = MutableReader(snapshot("gateway")) + offline = MutableReader(snapshot("cloud", health=ProviderHealth.OFFLINE)) + + def broken(): + """Simulate an integration-local reader failure.""" + raise RuntimeError("cloud API failed") + + compiler = LatticeAutoConfigCompiler({"gateway": good, "cloud": offline, "broken": broken}) + run = compiler.drain() + + self.assertEqual(run.status, CompileStatus.DEGRADED) + self.assertEqual(dict(run.plan.provider_generations), {"gateway": 1}) + self.assertEqual({issue.code for issue in run.issues}, {"provider_offline", "provider_read_failed"}) + self.assertEqual((good.calls, offline.calls), (1, 1)) + + def test_malformed_provider_is_isolated(self): + """Malformed topology from one integration does not poison healthy input.""" + malformed = ProviderSnapshot("cloud", 1, ProviderHealth.HEALTHY, {"not": "topology"}) + compiler = LatticeAutoConfigCompiler({"gateway": MutableReader(snapshot("gateway")), "cloud": MutableReader(malformed)}) + run = compiler.drain() + + self.assertEqual(run.status, CompileStatus.DEGRADED) + self.assertEqual(dict(run.plan.provider_generations), {"gateway": 1}) + self.assertIn("provider_invalid", {issue.code for issue in run.issues}) + + def test_compile_failure_preserves_last_known_good_as_stale(self): + """A later global collision leaves the prior immutable plan active.""" + reader = MutableReader(snapshot("gateway", generation=1)) + compiler = LatticeAutoConfigCompiler({"gateway": reader}) + first = compiler.drain() + last_known_good = first.plan + + bad_fragment = fragment("gateway", 2) + bad_fragment["nodes"].append(dict(bad_fragment["nodes"][0])) + reader.value = ProviderSnapshot("gateway", 2, ProviderHealth.HEALTHY, bad_fragment) + self.assertTrue(compiler.invalidate("gateway", 2, "conflicting rediscovery")) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertIs(failed.plan, last_known_good) + self.assertIn("compile_failed", {issue.code for issue in failed.issues}) + + def test_all_offline_preserves_last_known_good_as_stale(self): + """Temporary total provider loss cannot replace a working plan.""" + reader = MutableReader(snapshot("gateway", generation=1)) + compiler = LatticeAutoConfigCompiler({"gateway": reader}) + last_known_good = compiler.drain().plan + + reader.value = snapshot("gateway", generation=2, health=ProviderHealth.OFFLINE) + self.assertTrue(compiler.invalidate("gateway", 2, "connection lost")) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertIs(failed.plan, last_known_good) + self.assertEqual( + {issue.code for issue in failed.issues}, + {"provider_offline", "active_provider_unavailable", "compile_failed"}, + ) + self.assertTrue(failed.pending) + + def test_unavailable_active_provider_cannot_materialize_destructive_removal(self): + """A transient provider outage keeps the complete last-known-good plan.""" + gateway = MutableReader(snapshot("gateway", generation=1, node_id="GW1")) + cloud = MutableReader(snapshot("cloud", generation=1, node_id="CLOUD1")) + requests = [] + compiler = LatticeAutoConfigCompiler( + {"gateway": gateway, "cloud": cloud}, + requests.append, + ) + first = compiler.drain() + last_known_good = first.plan + + cloud.value = snapshot( + "cloud", + generation=2, + node_id="CLOUD1", + health=ProviderHealth.OFFLINE, + ) + self.assertTrue(compiler.invalidate("cloud", 2, "cloud unavailable")) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertIs(failed.plan, last_known_good) + self.assertEqual(len(requests), 1) + self.assertEqual( + set(dict(failed.plan.provider_generations)), + {"gateway", "cloud"}, + ) + self.assertIn( + "active_provider_unavailable", + {issue.code for issue in failed.issues}, + ) + self.assertTrue(failed.pending) + + def test_materialization_failure_is_retryable_without_new_generation(self): + """Caller-driven retry can materialize the same complete generation.""" + reader = MutableReader(snapshot("gateway", generation=1)) + requests = [] + + def materialize(request): + """Fail once, then accept the exact same semantic plan.""" + requests.append(request) + if len(requests) == 1: + raise RuntimeError("temporary config store failure") + + compiler = LatticeAutoConfigCompiler({"gateway": reader}, materialize) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertIsNone(failed.plan) + self.assertEqual(failed.materializations, 0) + self.assertTrue(failed.pending) + self.assertIn( + "materialization_failed", + {issue.code for issue in failed.issues}, + ) + + recovered = compiler.drain() + + self.assertEqual(recovered.status, CompileStatus.FRESH) + self.assertIsNotNone(recovered.plan) + self.assertEqual(recovered.materializations, 1) + self.assertFalse(recovered.pending) + self.assertEqual(reader.calls, 2) + self.assertEqual(len(requests), 2) + self.assertEqual(requests[0].plan.digest, requests[1].plan.digest) + + def test_unchanged_digest_skips_materialization(self): + """A newer generation with identical semantics updates provenance only.""" + reader = MutableReader(snapshot("gateway", generation=1)) + requests = [] + compiler = LatticeAutoConfigCompiler({"gateway": reader}, requests.append) + first = compiler.drain() + + reader.value = snapshot("gateway", generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "heartbeat refresh")) + second = compiler.drain() + + self.assertEqual(first.materializations, 1) + self.assertEqual(second.materializations, 0) + self.assertEqual(len(requests), 1) + self.assertEqual(dict(second.plan.provider_generations), {"gateway": 2}) + + def test_materializer_feedback_token_cannot_recompile(self): + """A materializer-caused integration event is not a feedback loop.""" + reader = MutableReader(snapshot("gateway", generation=1)) + feedback_results = [] + holder = {} + + def materialize(request): + """Echo the materializer token through the provider invalidation API.""" + feedback_results.append(holder["compiler"].invalidate("gateway", 2, "materialized config observed", request.feedback_token)) + + compiler = LatticeAutoConfigCompiler({"gateway": reader}, materialize) + holder["compiler"] = compiler + run = compiler.drain() + + self.assertEqual(feedback_results, [False]) + self.assertEqual(run.attempts, 1) + self.assertEqual(run.materializations, 1) + self.assertFalse(run.pending) + + def test_every_attempt_fresh_reads_all_providers(self): + """Independent invalidations still re-read the complete provider set.""" + gateway = MutableReader(snapshot("gateway", generation=1)) + cloud = MutableReader(snapshot("cloud", generation=1)) + compiler = LatticeAutoConfigCompiler({"gateway": gateway, "cloud": cloud}) + compiler.drain() + + gateway.value = snapshot("gateway", generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "new gateway topology")) + compiler.drain() + + self.assertEqual((gateway.calls, cloud.calls), (2, 2)) + + +if __name__ == "__main__": + unittest.main() From 3979dac6fa61af6177b64a31bfcbe0ee3be8a4b6 Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Mon, 27 Jul 2026 23:03:26 +0100 Subject: [PATCH 2/8] feat: compile indexed lattice roles --- apps/predbat/lattice_autoconfig.py | 310 +++++++++++++++- apps/predbat/tests/test_lattice_autoconfig.py | 349 ++++++++++++++++-- 2 files changed, 607 insertions(+), 52 deletions(-) diff --git a/apps/predbat/lattice_autoconfig.py b/apps/predbat/lattice_autoconfig.py index 66922793e..847ff438f 100644 --- a/apps/predbat/lattice_autoconfig.py +++ b/apps/predbat/lattice_autoconfig.py @@ -98,6 +98,29 @@ def __post_init__(self): object.__setattr__(self, "node_id", self.node_id.strip()) +@dataclass(frozen=True) +class ProviderRoleAssignment: + """A provider-local indexed primary or control role assertion.""" + + role: AliasRole + group: str + index: int + node_id: str + + def __post_init__(self): + """Validate and normalise one indexed role assertion.""" + if self.role not in (AliasRole.PRIMARY, AliasRole.CONTROL): + raise ValueError("role assignment must be PRIMARY or CONTROL") + if not isinstance(self.group, str) or not self.group.strip(): + raise ValueError("role assignment group must be a non-empty string") + if not isinstance(self.index, int) or isinstance(self.index, bool) or self.index < 0: + raise ValueError("role assignment index must be a non-negative integer") + if not isinstance(self.node_id, str) or not self.node_id.strip(): + raise ValueError("role assignment node_id must be a non-empty string") + object.__setattr__(self, "group", self.group.strip()) + object.__setattr__(self, "node_id", self.node_id.strip()) + + @dataclass(frozen=True) class ProviderSnapshot: """One integration's immutable generation of health, topology, and aliases.""" @@ -108,6 +131,7 @@ class ProviderSnapshot: topology_fragment: Mapping aliases: tuple = () identity_aliases: tuple = () + role_assignments: tuple = () def __post_init__(self): """Validate scalar fields and detach caller-owned mutable data.""" @@ -125,10 +149,14 @@ def __post_init__(self): identity_aliases = tuple(self.identity_aliases) if any(not isinstance(alias, ProviderIdentityAlias) for alias in identity_aliases): raise ValueError("identity_aliases must contain ProviderIdentityAlias values") + role_assignments = tuple(self.role_assignments) + if any(not isinstance(assignment, ProviderRoleAssignment) for assignment in role_assignments): + raise ValueError("role_assignments must contain ProviderRoleAssignment values") object.__setattr__(self, "provider_id", self.provider_id.strip()) object.__setattr__(self, "topology_fragment", _freeze(copy.deepcopy(dict(self.topology_fragment)))) object.__setattr__(self, "aliases", aliases) object.__setattr__(self, "identity_aliases", identity_aliases) + object.__setattr__(self, "role_assignments", role_assignments) @dataclass(frozen=True) @@ -154,6 +182,58 @@ class IdentityBinding: canonical_node_id: str +@dataclass(frozen=True) +class RoleAssignmentBinding: + """One normalized provider assertion for an indexed plan role.""" + + provider_id: str + generation: int + role: str + group: str + index: int + local_node_id: str + canonical_node_id: str + + +@dataclass(frozen=True) +class IndexedRoleTarget: + """One deterministic indexed primary or control target.""" + + group: str + index: int + node_id: str + provenance: tuple + + +@dataclass(frozen=True) +class MaterializationReadiness: + """Fail-closed decision exposed to a future config materializer.""" + + ready: bool + blockers: tuple + + def __post_init__(self): + """Normalise blockers and reject contradictory readiness state.""" + if not isinstance(self.ready, bool): + raise ValueError("materialization readiness must be a boolean") + if isinstance(self.blockers, str): + raise ValueError("materialization blockers must be an iterable of strings") + try: + blockers = tuple(self.blockers) + except TypeError as exc: + raise ValueError("materialization blockers must be an iterable of strings") from exc + if any(not isinstance(blocker, str) for blocker in blockers): + raise ValueError("materialization blockers must be non-empty strings") + blockers = tuple(blocker.strip() for blocker in blockers) + if any(not blocker for blocker in blockers): + raise ValueError("materialization blockers must be non-empty strings") + if len(blockers) != len(set(blockers)): + raise ValueError("materialization blockers must be unique") + if self.ready != (not blockers): + raise ValueError("materialization readiness must equal absence of blockers") + object.__setattr__(self, "blockers", blockers) + + @dataclass(frozen=True) class FieldProvenance: """Source coordinates for one generated plan field.""" @@ -181,6 +261,12 @@ class AutoConfigPlan: topology: Mapping aliases: tuple identity_aliases: tuple + role_assignments: tuple + primary_targets: tuple + control_targets: tuple + primary_target: Optional[str] + control_target: Optional[str] + materialization_readiness: MaterializationReadiness fields: tuple provenance: tuple provider_generations: tuple @@ -282,6 +368,15 @@ def _fingerprint_snapshot(snapshot): } for alias in sorted(snapshot.identity_aliases, key=lambda item: (item.kind, item.value, item.node_id)) ] + role_assignments = [ + { + "role": assignment.role.value, + "group": assignment.group, + "index": assignment.index, + "node_id": assignment.node_id, + } + for assignment in sorted(snapshot.role_assignments, key=lambda item: (item.group, item.role.value, item.index, item.node_id)) + ] payload = { "provider": snapshot.provider_id, "generation": snapshot.generation, @@ -289,6 +384,7 @@ def _fingerprint_snapshot(snapshot): "fragment": snapshot.topology_fragment, "aliases": aliases, "identity_aliases": identity_aliases, + "role_assignments": role_assignments, } return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() @@ -449,7 +545,154 @@ def _correlate_identities(snapshots, documents, provider_nodes): return tuple(normalized_documents), canonical_by_node, identity_bindings -def _field_provenance(snapshots, bindings, identity_bindings, primary_target, control_target, topology_snapshot, canonical_by_node): +def _compile_roles(snapshots, bindings, identity_bindings, provider_nodes, canonical_by_node): + """Validate legacy/indexed roles and return deterministic target outputs.""" + legacy_by_role = {role: tuple(binding for binding in bindings if role.value in binding.roles) for role in (AliasRole.PRIMARY, AliasRole.CONTROL)} + legacy_assignments = tuple(binding for role_bindings in legacy_by_role.values() for binding in role_bindings) + + role_bindings = [] + qualified_assignments = set() + generation_by_provider = {snapshot.provider_id: snapshot.generation for snapshot in snapshots} + for snapshot in snapshots: + assignments = sorted(snapshot.role_assignments, key=lambda item: (item.group, item.role.value, item.index, item.node_id)) + for assignment in assignments: + if assignment.node_id not in provider_nodes[snapshot.provider_id]: + raise AutoConfigCompileError( + "role assignment {}:{}:{} targets unknown provider-local node {}".format( + assignment.group, + assignment.role.value, + assignment.index, + assignment.node_id, + ) + ) + qualified_key = (snapshot.provider_id, assignment.group, assignment.role.value, assignment.index) + if qualified_key in qualified_assignments: + raise AutoConfigCompileError( + "role assignment collision for {}:{}:{}:{}".format( + snapshot.provider_id, + assignment.group, + assignment.role.value, + assignment.index, + ) + ) + qualified_assignments.add(qualified_key) + role_bindings.append( + RoleAssignmentBinding( + provider_id=snapshot.provider_id, + generation=snapshot.generation, + role=assignment.role.value, + group=assignment.group, + index=assignment.index, + local_node_id=assignment.node_id, + canonical_node_id=canonical_by_node[(snapshot.provider_id, assignment.node_id)], + ) + ) + role_bindings = tuple( + sorted( + role_bindings, + key=lambda item: (item.group, item.role, item.index, item.provider_id, item.local_node_id), + ) + ) + + if role_bindings and legacy_assignments: + raise AutoConfigCompileError("legacy and indexed role assignments cannot be mixed") + + legacy_targets = {} + if not role_bindings: + for role, role_aliases in legacy_by_role.items(): + if len(role_aliases) > 1: + raise AutoConfigCompileError("ambiguous legacy {} assignments".format(role.value)) + legacy_targets[role] = role_aliases[0].node_id if role_aliases else None + else: + legacy_targets = {AliasRole.PRIMARY: None, AliasRole.CONTROL: None} + + indices_by_group_role = {} + assignments_by_slot = {} + for binding in role_bindings: + group_role = (binding.group, binding.role) + indices_by_group_role.setdefault(group_role, set()).add(binding.index) + assignments_by_slot.setdefault((binding.group, binding.role, binding.index), []).append(binding) + for (group, role), indices in sorted(indices_by_group_role.items()): + ordered = sorted(indices) + expected = list(range(ordered[-1] + 1)) + if ordered != expected: + raise AutoConfigCompileError( + "{} {} indices must be contiguous from zero; got {}".format( + group, + role, + ordered, + ) + ) + + correlated_providers = {} + for binding in identity_bindings: + correlated_providers.setdefault(binding.canonical_node_id, set()).add(binding.provider_id) + + indexed_targets = {AliasRole.PRIMARY: [], AliasRole.CONTROL: []} + for (group, role_value, index), assignments in sorted(assignments_by_slot.items()): + canonical_nodes = {assignment.canonical_node_id for assignment in assignments} + if len(canonical_nodes) != 1: + raise AutoConfigCompileError( + "conflicting {} target for {} index {}: {}".format( + role_value, + group, + index, + sorted(canonical_nodes), + ) + ) + canonical_node_id = next(iter(canonical_nodes)) + providers = {assignment.provider_id for assignment in assignments} + if len(providers) > 1 and not providers.issubset(correlated_providers.get(canonical_node_id, set())): + raise AutoConfigCompileError( + "providers sharing {} {} index {} require explicit strong identity correlation".format( + group, + role_value, + index, + ) + ) + provenance = tuple( + FieldProvenance( + "/{}_targets/{}/{}/node_id".format(role_value, group, index), + assignment.provider_id, + generation_by_provider[assignment.provider_id], + "/role_assignments/{}/{}/{}".format(role_value, group, index), + ) + for assignment in assignments + ) + indexed_targets[AliasRole(role_value)].append( + IndexedRoleTarget( + group=group, + index=index, + node_id=canonical_node_id, + provenance=provenance, + ) + ) + + primary_targets = tuple(indexed_targets[AliasRole.PRIMARY]) + control_targets = tuple(indexed_targets[AliasRole.CONTROL]) + blockers = [] + if not primary_targets: + blockers.append("indexed_primary_targets_missing") + if not control_targets: + blockers.append("indexed_control_targets_missing") + if legacy_assignments: + blockers.append("legacy_role_assignments_present") + blockers.append("config_projection_bindings_missing") + readiness = MaterializationReadiness(ready=False, blockers=tuple(blockers)) + return role_bindings, primary_targets, control_targets, legacy_targets, readiness + + +def _field_provenance( + snapshots, + bindings, + identity_bindings, + primary_target, + control_target, + primary_targets, + control_targets, + topology_snapshot, + canonical_by_node, +): """Build deterministic source coordinates for every generated field.""" provenance = [] for snapshot in snapshots: @@ -485,6 +728,8 @@ def _field_provenance(snapshots, bindings, identity_bindings, primary_target, co "/aliases/{}".format(binding.qualified_name.split(":", 1)[1]), ) ) + for target in primary_targets + control_targets: + provenance.extend(target.provenance) generations = {snapshot.provider_id: snapshot.generation for snapshot in snapshots} local_by_canonical = {(provider_id, canonical_node_id): local_node_id for (provider_id, local_node_id), canonical_node_id in canonical_by_node.items()} for key, source in topology_snapshot.provenance.items(): @@ -544,12 +789,15 @@ def compile_auto_config(snapshots): ) bindings = tuple(sorted(bindings, key=lambda item: item.qualified_name)) - targets = {} - for role in (AliasRole.PRIMARY, AliasRole.CONTROL): - role_targets = sorted({binding.node_id for binding in bindings if role.value in binding.roles}) - if len(role_targets) > 1: - raise AutoConfigCompileError("ambiguous {} target: {}".format(role.value, ", ".join(role_targets))) - targets[role] = role_targets[0] if role_targets else None + role_bindings, primary_targets, control_targets, legacy_targets, readiness = _compile_roles( + snapshots, + bindings, + identity_bindings, + provider_nodes, + canonical_by_node, + ) + primary_target = legacy_targets[AliasRole.PRIMARY] + control_target = legacy_targets[AliasRole.CONTROL] topology_snapshot = merge_topologies(documents) fields = [] @@ -561,8 +809,10 @@ def compile_auto_config(snapshots): "/aliases/{}".format(binding.qualified_name.split(":", 1)[1]), ) fields.append(AutoConfigField("alias.{}".format(binding.qualified_name), binding.node_id, (source,))) - for name, role in (("primary_target", AliasRole.PRIMARY), ("control_target", AliasRole.CONTROL)): - target = targets[role] + for name, target, role in ( + ("primary_target", primary_target, AliasRole.PRIMARY), + ("control_target", control_target, AliasRole.CONTROL), + ): if target is None: continue sources = tuple( @@ -576,6 +826,15 @@ def compile_auto_config(snapshots): if binding.node_id == target and role.value in binding.roles ) fields.append(AutoConfigField(name, target, sources)) + for name, targets in (("primary_targets", primary_targets), ("control_targets", control_targets)): + for target in targets: + fields.append( + AutoConfigField( + "{}.{}.{}".format(name, target.group, target.index), + target.node_id, + target.provenance, + ) + ) fields = tuple(sorted(fields, key=lambda item: item.name)) semantic = { @@ -598,6 +857,25 @@ def compile_auto_config(snapshots): } for binding in identity_bindings ], + "role_assignments": [ + { + "provider_id": binding.provider_id, + "role": binding.role, + "group": binding.group, + "index": binding.index, + "local_node_id": binding.local_node_id, + "canonical_node_id": binding.canonical_node_id, + } + for binding in role_bindings + ], + "primary_targets": [{"group": target.group, "index": target.index, "node_id": target.node_id} for target in primary_targets], + "control_targets": [{"group": target.group, "index": target.index, "node_id": target.node_id} for target in control_targets], + "primary_target": primary_target, + "control_target": control_target, + "materialization_readiness": { + "ready": readiness.ready, + "blockers": readiness.blockers, + }, "fields": [{"name": field.name, "value": field.value} for field in fields], } digest = hashlib.sha256(_canonical_json(semantic).encode("utf-8")).hexdigest() @@ -605,8 +883,10 @@ def compile_auto_config(snapshots): snapshots, bindings, identity_bindings, - targets[AliasRole.PRIMARY], - targets[AliasRole.CONTROL], + primary_target, + control_target, + primary_targets, + control_targets, topology_snapshot, canonical_by_node, ) @@ -615,6 +895,12 @@ def compile_auto_config(snapshots): topology=_freeze(topology_snapshot.site), aliases=bindings, identity_aliases=identity_bindings, + role_assignments=role_bindings, + primary_targets=primary_targets, + control_targets=control_targets, + primary_target=primary_target, + control_target=control_target, + materialization_readiness=readiness, fields=fields, provenance=provenance, provider_generations=tuple((snapshot.provider_id, snapshot.generation) for snapshot in snapshots), @@ -758,7 +1044,7 @@ def _compile_attempt(self): def _materialize_if_changed(self, plan): """Hand a changed plan to the injected materializer exactly once.""" with self._lock: - if self._materializer is None or (self._active_plan is not None and plan.digest == self._active_plan.digest): + if not plan.materialization_readiness.ready or self._materializer is None or (self._active_plan is not None and plan.digest == self._active_plan.digest): return 0, () self._token_counter += 1 feedback_token = "lattice-autoconfig-{}".format(self._token_counter) diff --git a/apps/predbat/tests/test_lattice_autoconfig.py b/apps/predbat/tests/test_lattice_autoconfig.py index 5a1d14aab..285fb4010 100644 --- a/apps/predbat/tests/test_lattice_autoconfig.py +++ b/apps/predbat/tests/test_lattice_autoconfig.py @@ -6,6 +6,7 @@ import sys import threading import unittest +from dataclasses import replace sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) @@ -14,9 +15,11 @@ AutoConfigCompileError, CompileStatus, LatticeAutoConfigCompiler, + MaterializationReadiness, ProviderAlias, ProviderHealth, ProviderIdentityAlias, + ProviderRoleAssignment, ProviderSnapshot, compile_auto_config, ) @@ -50,9 +53,35 @@ def fragment(provider, generation, node_id="INV1", kind="inverter"): } -def snapshot(provider, generation=1, node_id="INV1", kind="inverter", health=ProviderHealth.HEALTHY, aliases=(), identity_aliases=()): +def snapshot( + provider, + generation=1, + node_id="INV1", + kind="inverter", + health=ProviderHealth.HEALTHY, + aliases=(), + identity_aliases=(), + role_assignments=(), +): """Build one typed provider snapshot.""" - return ProviderSnapshot(provider, generation, health, fragment(provider, generation, node_id=node_id, kind=kind), aliases, identity_aliases) + return ProviderSnapshot( + provider, + generation, + health, + fragment(provider, generation, node_id=node_id, kind=kind), + aliases, + identity_aliases, + role_assignments, + ) + + +def ready_plan(): + """Copy a compiled shadow plan into a future-materializer test harness.""" + plan = compile_auto_config((snapshot("gateway"),)) + return replace( + plan, + materialization_readiness=MaterializationReadiness(True, ()), + ) class MutableReader: @@ -72,11 +101,34 @@ def __call__(self): class TestPlanCompilation(unittest.TestCase): """Compiler output is safe, immutable, deterministic, and attributable.""" + def test_materialization_readiness_rejects_inconsistent_state(self): + """Readiness is derived exactly from a normalized blocker tuple.""" + readiness = MaterializationReadiness( + False, + [" config_projection_bindings_missing "], + ) + + self.assertEqual( + readiness.blockers, + ("config_projection_bindings_missing",), + ) + with self.assertRaisesRegex(ValueError, "absence of blockers"): + MaterializationReadiness(True, ("projection_missing",)) + with self.assertRaisesRegex(ValueError, "absence of blockers"): + MaterializationReadiness(False, ()) + with self.assertRaisesRegex(ValueError, "unique"): + MaterializationReadiness(False, ("projection_missing", "projection_missing")) + with self.assertRaisesRegex(ValueError, "non-empty"): + MaterializationReadiness(False, (" ",)) + with self.assertRaisesRegex(ValueError, "iterable of strings"): + MaterializationReadiness(False, "projection_missing") + def test_order_independent_digest_and_provider_qualified_aliases(self): """Input order and shared local alias names cannot alter a plan.""" - alias = ProviderAlias("battery", "INV1", frozenset((AliasRole.REFERENCE, AliasRole.PRIMARY, AliasRole.CONTROL))) - gateway = snapshot("gateway", aliases=(alias,), identity_aliases=(ProviderIdentityAlias("serial", "SER123", "INV1"),)) - cloud = snapshot("cloud", aliases=(alias,), identity_aliases=(ProviderIdentityAlias("serial", "SER123", "INV1"),)) + gateway_alias = ProviderAlias("battery", "INV1", frozenset((AliasRole.REFERENCE, AliasRole.PRIMARY, AliasRole.CONTROL))) + cloud_alias = ProviderAlias("battery", "INV1") + gateway = snapshot("gateway", aliases=(gateway_alias,), identity_aliases=(ProviderIdentityAlias("serial", "SER123", "INV1"),)) + cloud = snapshot("cloud", aliases=(cloud_alias,), identity_aliases=(ProviderIdentityAlias("serial", "SER123", "INV1"),)) left = compile_auto_config((gateway, cloud)) right = compile_auto_config((cloud, gateway)) @@ -85,6 +137,10 @@ def test_order_independent_digest_and_provider_qualified_aliases(self): self.assertEqual([binding.qualified_name for binding in left.aliases], ["cloud:battery", "gateway:battery"]) self.assertEqual(dict(left.provider_generations), {"cloud": 1, "gateway": 1}) self.assertEqual({field.name for field in left.fields}, {"alias.cloud:battery", "alias.gateway:battery", "control_target", "primary_target"}) + self.assertEqual(left.primary_target, "identity:serial:SER123") + self.assertEqual(left.control_target, "identity:serial:SER123") + self.assertEqual(left.primary_targets, ()) + self.assertEqual(left.control_targets, ()) self.assertTrue(all(field.provenance for field in left.fields)) with self.assertRaises(TypeError): left.topology["scope"] = "fragment" @@ -181,12 +237,12 @@ def test_ambiguous_primary_and_control_targets_fail_closed(self): """Several distinct target nodes cannot silently pick a winner.""" primary_a = ProviderAlias("battery", "INV1", frozenset((AliasRole.PRIMARY,))) primary_b = ProviderAlias("battery", "INV2", frozenset((AliasRole.PRIMARY,))) - with self.assertRaisesRegex(AutoConfigCompileError, "ambiguous primary"): + with self.assertRaisesRegex(AutoConfigCompileError, "ambiguous legacy primary"): compile_auto_config((snapshot("gateway", node_id="INV1", aliases=(primary_a,)), snapshot("cloud", node_id="INV2", aliases=(primary_b,)))) control_a = ProviderAlias("control", "INV1", frozenset((AliasRole.CONTROL,))) control_b = ProviderAlias("control", "INV2", frozenset((AliasRole.CONTROL,))) - with self.assertRaisesRegex(AutoConfigCompileError, "ambiguous control"): + with self.assertRaisesRegex(AutoConfigCompileError, "ambiguous legacy control"): compile_auto_config((snapshot("gateway", node_id="INV1", aliases=(control_a,)), snapshot("cloud", node_id="INV2", aliases=(control_b,)))) def test_alias_must_target_provider_local_identity(self): @@ -195,6 +251,157 @@ def test_alias_must_target_provider_local_identity(self): with self.assertRaisesRegex(AutoConfigCompileError, "unknown provider-local node"): compile_auto_config((snapshot("gateway", node_id="INV1", aliases=(bad,)), snapshot("cloud", node_id="INV2"))) + def test_indexed_roles_are_order_independent_and_contiguous(self): + """Indexed targets sort by group, role, and index regardless of input order.""" + first = snapshot( + "a", + node_id="A", + role_assignments=( + ProviderRoleAssignment(AliasRole.CONTROL, "battery", 1, "A"), + ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 0, "A"), + ), + ) + second = snapshot( + "z", + node_id="Z", + role_assignments=( + ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 1, "Z"), + ProviderRoleAssignment(AliasRole.CONTROL, "battery", 0, "Z"), + ), + ) + + left = compile_auto_config((second, first)) + right = compile_auto_config((first, second)) + + self.assertEqual(left.digest, right.digest) + self.assertEqual( + [(target.group, target.index, target.node_id) for target in left.primary_targets], + [ + ("battery", 0, "provider:a:A"), + ("battery", 1, "provider:z:Z"), + ], + ) + self.assertEqual( + [(target.group, target.index, target.node_id) for target in left.control_targets], + [ + ("battery", 0, "provider:z:Z"), + ("battery", 1, "provider:a:A"), + ], + ) + self.assertEqual( + [(item.group, item.role, item.index) for item in left.role_assignments], + sorted((item.group, item.role, item.index) for item in left.role_assignments), + ) + self.assertEqual( + {field.name for field in left.fields if field.name.startswith(("primary_targets", "control_targets"))}, + { + "primary_targets.battery.0", + "primary_targets.battery.1", + "control_targets.battery.0", + "control_targets.battery.1", + }, + ) + indexed_fields = [field for field in left.fields if field.name.startswith(("primary_targets", "control_targets"))] + self.assertTrue(all(field.provenance for field in indexed_fields)) + self.assertTrue(all(item in left.provenance for field in indexed_fields for item in field.provenance)) + self.assertFalse(left.materialization_readiness.ready) + self.assertEqual(left.materialization_readiness.blockers, ("config_projection_bindings_missing",)) + with self.assertRaises(AttributeError): + left.primary_targets[0].node_id = "changed" + + def test_provider_role_assignment_rejects_reference_and_negative_index(self): + """Only indexed primary/control assignments with non-negative indices exist.""" + with self.assertRaisesRegex(ValueError, "PRIMARY or CONTROL"): + ProviderRoleAssignment(AliasRole.REFERENCE, "battery", 0, "INV1") + with self.assertRaisesRegex(ValueError, "non-negative"): + ProviderRoleAssignment(AliasRole.PRIMARY, "battery", -1, "INV1") + + def test_indexed_role_indices_must_be_contiguous_per_group_and_role(self): + """A gap in one role sequence fails without affecting another sequence.""" + assignments = ( + ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 0, "INV1"), + ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 2, "INV1"), + ) + with self.assertRaisesRegex(AutoConfigCompileError, "indices must be contiguous"): + compile_auto_config((snapshot("gateway", role_assignments=assignments),)) + + def test_correlated_providers_may_share_one_index(self): + """Explicit strong identity correlation permits duplicate provider assertions.""" + gateway_identity = ProviderIdentityAlias("serial", "SER123", "gw") + cloud_identity = ProviderIdentityAlias("serial", "SER123", "cloud") + gateway_role = ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 0, "gw") + cloud_role = ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 0, "cloud") + plan = compile_auto_config( + ( + snapshot("gateway", node_id="gw", identity_aliases=(gateway_identity,), role_assignments=(gateway_role,)), + snapshot("cloud", node_id="cloud", identity_aliases=(cloud_identity,), role_assignments=(cloud_role,)), + ) + ) + + self.assertEqual(len(plan.primary_targets), 1) + self.assertEqual(plan.primary_targets[0].node_id, "identity:serial:SER123") + self.assertEqual({item.provider_id for item in plan.primary_targets[0].provenance}, {"gateway", "cloud"}) + + def test_uncorrelated_providers_conflicting_at_one_index_fail_closed(self): + """Equal role slots cannot select unrelated provider-local nodes.""" + gateway_role = ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 0, "gw") + cloud_role = ProviderRoleAssignment(AliasRole.PRIMARY, "battery", 0, "cloud") + with self.assertRaisesRegex(AutoConfigCompileError, "conflicting primary target"): + compile_auto_config( + ( + snapshot("gateway", node_id="gw", role_assignments=(gateway_role,)), + snapshot("cloud", node_id="cloud", role_assignments=(cloud_role,)), + ) + ) + + def test_same_node_may_fill_multiple_indices(self): + """An EMS aggregate can fan one canonical node out over several indices.""" + assignments = ( + ProviderRoleAssignment(AliasRole.PRIMARY, "ems", 0, "EMS"), + ProviderRoleAssignment(AliasRole.PRIMARY, "ems", 1, "EMS"), + ) + plan = compile_auto_config((snapshot("ge-cloud", node_id="EMS", role_assignments=assignments),)) + + self.assertEqual([target.node_id for target in plan.primary_targets], ["provider:ge-cloud:EMS", "provider:ge-cloud:EMS"]) + + def test_legacy_and_indexed_role_assignments_cannot_mix(self): + """A plan must use exactly one target-addressing model.""" + legacy = ProviderAlias("battery", "INV1", frozenset((AliasRole.PRIMARY,))) + indexed = ProviderRoleAssignment(AliasRole.CONTROL, "battery", 0, "INV1") + with self.assertRaisesRegex(AutoConfigCompileError, "cannot be mixed"): + compile_auto_config((snapshot("gateway", aliases=(legacy,), role_assignments=(indexed,)),)) + + def test_multiple_legacy_assignments_are_ambiguous_even_when_correlated(self): + """The singular compatibility field represents exactly one assertion.""" + gateway_alias = ProviderAlias("battery", "gw", frozenset((AliasRole.PRIMARY,))) + cloud_alias = ProviderAlias("battery", "cloud", frozenset((AliasRole.PRIMARY,))) + gateway_identity = ProviderIdentityAlias("serial", "SER123", "gw") + cloud_identity = ProviderIdentityAlias("serial", "SER123", "cloud") + with self.assertRaisesRegex(AutoConfigCompileError, "ambiguous legacy primary"): + compile_auto_config( + ( + snapshot("gateway", node_id="gw", aliases=(gateway_alias,), identity_aliases=(gateway_identity,)), + snapshot("cloud", node_id="cloud", aliases=(cloud_alias,), identity_aliases=(cloud_identity,)), + ) + ) + + def test_reference_only_plan_is_explicitly_shadow_only(self): + """Reference discovery compiles but exposes every write blocker.""" + reference = ProviderAlias("battery", "INV1") + plan = compile_auto_config((snapshot("gateway", aliases=(reference,)),)) + + self.assertFalse(plan.materialization_readiness.ready) + self.assertEqual( + plan.materialization_readiness.blockers, + ( + "indexed_primary_targets_missing", + "indexed_control_targets_missing", + "config_projection_bindings_missing", + ), + ) + self.assertIsNone(plan.primary_target) + self.assertIsNone(plan.control_target) + class TestInvalidationStateMachine(unittest.TestCase): """Invalidations coalesce without losing freshness or last-known-good state.""" @@ -265,8 +472,9 @@ def reader(): self.assertEqual(run.attempts, 2) self.assertEqual(state["calls"], 2) self.assertEqual(dict(run.plan.provider_generations), {"gateway": 2}) - self.assertEqual([dict(request.plan.provider_generations) for request in requests], [{"gateway": 2}]) - self.assertEqual(run.materializations, 1) + self.assertEqual(requests, []) + self.assertEqual(run.materializations, 0) + self.assertFalse(run.plan.materialization_readiness.ready) self.assertFalse(run.pending) def test_all_accepted_invalidation_causes_survive_coalescing(self): @@ -395,7 +603,7 @@ def test_unavailable_active_provider_cannot_materialize_destructive_removal(self self.assertEqual(failed.status, CompileStatus.STALE) self.assertIs(failed.plan, last_known_good) - self.assertEqual(len(requests), 1) + self.assertEqual(len(requests), 0) self.assertEqual( set(dict(failed.plan.provider_generations)), {"gateway", "cloud"}, @@ -406,38 +614,24 @@ def test_unavailable_active_provider_cannot_materialize_destructive_removal(self ) self.assertTrue(failed.pending) - def test_materialization_failure_is_retryable_without_new_generation(self): - """Caller-driven retry can materialize the same complete generation.""" + def test_shadow_only_plan_never_invokes_materializer(self): + """A not-ready plan remains observable without reaching a write callback.""" reader = MutableReader(snapshot("gateway", generation=1)) requests = [] def materialize(request): - """Fail once, then accept the exact same semantic plan.""" + """Record any unsafe hand-off to make the test fail.""" requests.append(request) - if len(requests) == 1: - raise RuntimeError("temporary config store failure") compiler = LatticeAutoConfigCompiler({"gateway": reader}, materialize) - failed = compiler.drain() - - self.assertEqual(failed.status, CompileStatus.STALE) - self.assertIsNone(failed.plan) - self.assertEqual(failed.materializations, 0) - self.assertTrue(failed.pending) - self.assertIn( - "materialization_failed", - {issue.code for issue in failed.issues}, - ) - - recovered = compiler.drain() + run = compiler.drain() - self.assertEqual(recovered.status, CompileStatus.FRESH) - self.assertIsNotNone(recovered.plan) - self.assertEqual(recovered.materializations, 1) - self.assertFalse(recovered.pending) - self.assertEqual(reader.calls, 2) - self.assertEqual(len(requests), 2) - self.assertEqual(requests[0].plan.digest, requests[1].plan.digest) + self.assertEqual(run.status, CompileStatus.FRESH) + self.assertIsNotNone(run.plan) + self.assertFalse(run.plan.materialization_readiness.ready) + self.assertEqual(run.materializations, 0) + self.assertFalse(run.pending) + self.assertEqual(requests, []) def test_unchanged_digest_skips_materialization(self): """A newer generation with identical semantics updates provenance only.""" @@ -450,13 +644,14 @@ def test_unchanged_digest_skips_materialization(self): self.assertTrue(compiler.invalidate("gateway", 2, "heartbeat refresh")) second = compiler.drain() - self.assertEqual(first.materializations, 1) + self.assertEqual(first.materializations, 0) self.assertEqual(second.materializations, 0) - self.assertEqual(len(requests), 1) + self.assertEqual(len(requests), 0) + self.assertEqual(first.plan.digest, second.plan.digest) self.assertEqual(dict(second.plan.provider_generations), {"gateway": 2}) - def test_materializer_feedback_token_cannot_recompile(self): - """A materializer-caused integration event is not a feedback loop.""" + def test_shadow_plan_cannot_create_materializer_feedback(self): + """A blocked hand-off cannot cause an integration feedback loop.""" reader = MutableReader(snapshot("gateway", generation=1)) feedback_results = [] holder = {} @@ -469,11 +664,85 @@ def materialize(request): holder["compiler"] = compiler run = compiler.drain() - self.assertEqual(feedback_results, [False]) + self.assertEqual(feedback_results, []) self.assertEqual(run.attempts, 1) - self.assertEqual(run.materializations, 1) + self.assertEqual(run.materializations, 0) self.assertFalse(run.pending) + def test_ready_plan_retries_after_materializer_failure(self): + """A failed hand-off is not treated as a successful materialization.""" + requests = [] + + def materialize(request): + """Fail the first hand-off and accept the retry.""" + requests.append(request) + if len(requests) == 1: + raise RuntimeError("temporary write failure") + + compiler = LatticeAutoConfigCompiler(materializer=materialize) + plan = ready_plan() + + count, issues = compiler._materialize_if_changed(plan) + self.assertEqual(count, 0) + self.assertEqual([issue.code for issue in issues], ["materialization_failed"]) + + count, issues = compiler._materialize_if_changed(plan) + self.assertEqual(count, 1) + self.assertEqual(issues, ()) + self.assertEqual(len(requests), 2) + + def test_ready_plan_unchanged_digest_skips_materialization(self): + """Bookkeeping-only generation changes do not repeat a ready hand-off.""" + requests = [] + compiler = LatticeAutoConfigCompiler(materializer=requests.append) + plan = ready_plan() + + count, issues = compiler._materialize_if_changed(plan) + self.assertEqual((count, issues), (1, ())) + compiler._active_plan = plan + newer_generation = replace( + plan, + provider_generations=(("gateway", 2),), + ) + + count, issues = compiler._materialize_if_changed(newer_generation) + self.assertEqual((count, issues), (0, ())) + self.assertEqual(len(requests), 1) + + def test_ready_plan_suppresses_materializer_feedback_token(self): + """A ready hand-off cannot invalidate itself through its feedback token.""" + feedback_results = [] + holder = {} + + def materialize(request): + """Echo the hand-off token through the provider invalidation API.""" + feedback_results.append( + holder["compiler"].invalidate( + "gateway", + 2, + "materialized config observed", + request.feedback_token, + ) + ) + + compiler = LatticeAutoConfigCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + materialize, + ) + holder["compiler"] = compiler + + count, issues = compiler._materialize_if_changed(ready_plan()) + + self.assertEqual((count, issues), (1, ())) + self.assertEqual(feedback_results, [False]) + self.assertTrue( + compiler.invalidate( + "gateway", + 2, + "independent provider change", + ) + ) + def test_every_attempt_fresh_reads_all_providers(self): """Independent invalidations still re-read the complete provider set.""" gateway = MutableReader(snapshot("gateway", generation=1)) From 1377cd67e6bf4a34c76990993784480eea05f6df Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Mon, 27 Jul 2026 23:30:23 +0100 Subject: [PATCH 3/8] feat: project lattice fragments into auto config --- apps/predbat/lattice_autoconfig.py | 643 ++++++++++++++- apps/predbat/tests/test_lattice_autoconfig.py | 770 ++++++++++++++++++ 2 files changed, 1411 insertions(+), 2 deletions(-) diff --git a/apps/predbat/lattice_autoconfig.py b/apps/predbat/lattice_autoconfig.py index 847ff438f..42a24c957 100644 --- a/apps/predbat/lattice_autoconfig.py +++ b/apps/predbat/lattice_autoconfig.py @@ -42,6 +42,28 @@ class AliasRole(Enum): CONTROL = "control" +class ProjectionValueKind(Enum): + """Kind of one provider-published PredBat configuration value.""" + + ENTITY = "entity" + CONSTANT = "constant" + NONE = "none" + + +class ProjectionRouting(Enum): + """Relationship between projected values and selected indexed targets.""" + + LEAF = "leaf" + COORDINATOR = "coordinator" + + +class ProjectionCardinality(Enum): + """Shape of one projected PredBat configuration argument.""" + + SCALAR = "scalar" + PER_INDEX = "per_index" + + class CompileStatus(Enum): """Observable compiler state after a drain.""" @@ -121,6 +143,136 @@ def __post_init__(self): object.__setattr__(self, "node_id", self.node_id.strip()) +@dataclass(frozen=True) +class ProviderProjectionValue: + """One ordered provider-local entity, constant, or explicit None value.""" + + node_id: str + kind: ProjectionValueKind + value: object = None + capability: Optional[str] = None + identity_kind: Optional[str] = None + identity_value: Optional[str] = None + access_path_id: Optional[str] = None + + def __post_init__(self): + """Validate the value and normalize its optional source selectors.""" + if not isinstance(self.node_id, str) or not self.node_id.strip(): + raise ValueError("projection value node_id must be a non-empty string") + if not isinstance(self.kind, ProjectionValueKind): + raise ValueError("projection value kind must be a ProjectionValueKind") + if self.kind is ProjectionValueKind.ENTITY: + if not isinstance(self.value, str) or not self.value.strip(): + raise ValueError("projection entity value must be a non-empty string") + if not isinstance(self.capability, str) or not self.capability.strip(): + raise ValueError("projection entity value must name a capability") + value = self.value.strip() + elif self.kind is ProjectionValueKind.CONSTANT: + if not isinstance(self.value, (str, int, float, bool)) or self.value is None: + raise ValueError("projection constant must be a non-None JSON scalar") + value = self.value + _canonical_json(value) + else: + if self.value is not None: + raise ValueError("projection None value must carry value=None") + value = None + + capability = self.capability + if capability is not None: + if not isinstance(capability, str) or not capability.strip(): + raise ValueError("projection capability must be a non-empty string") + capability = capability.strip() + + identity_kind = self.identity_kind + identity_value = self.identity_value + if (identity_kind is None) != (identity_value is None): + raise ValueError("projection identity selector requires both kind and value") + if identity_kind is not None: + if not isinstance(identity_kind, str) or not identity_kind.strip(): + raise ValueError("projection identity kind must be a non-empty string") + if not isinstance(identity_value, str) or not identity_value.strip(): + raise ValueError("projection identity value must be a non-empty string") + identity_kind = identity_kind.strip().lower() + identity_value = identity_value.strip() + + access_path_id = self.access_path_id + if access_path_id is not None: + if not isinstance(access_path_id, str) or not access_path_id.strip(): + raise ValueError("projection access_path_id must be a non-empty string") + access_path_id = access_path_id.strip() + + object.__setattr__(self, "node_id", self.node_id.strip()) + object.__setattr__(self, "value", value) + object.__setattr__(self, "capability", capability) + object.__setattr__(self, "identity_kind", identity_kind) + object.__setattr__(self, "identity_value", identity_value) + object.__setattr__(self, "access_path_id", access_path_id) + + +@dataclass(frozen=True) +class ProviderConfigProjection: + """Generic provider contract for one PredBat configuration argument.""" + + argument: str + role: AliasRole + group: str + routing: ProjectionRouting + cardinality: ProjectionCardinality + values: tuple + required: bool = True + transforms: tuple = () + + def __post_init__(self): + """Validate and normalize one immutable projection declaration.""" + if not isinstance(self.argument, str) or not self.argument.strip(): + raise ValueError("projection argument must be a non-empty string") + if self.role not in (AliasRole.PRIMARY, AliasRole.CONTROL): + raise ValueError("projection role must be PRIMARY or CONTROL") + if not isinstance(self.group, str) or not self.group.strip(): + raise ValueError("projection group must be a non-empty string") + if not isinstance(self.routing, ProjectionRouting): + raise ValueError("projection routing must be a ProjectionRouting") + if not isinstance(self.cardinality, ProjectionCardinality): + raise ValueError("projection cardinality must be a ProjectionCardinality") + if not isinstance(self.required, bool): + raise ValueError("projection required must be a boolean") + values = tuple(self.values) + if not values or any(not isinstance(value, ProviderProjectionValue) for value in values): + raise ValueError("projection values must contain ProviderProjectionValue values") + if self.cardinality is ProjectionCardinality.SCALAR and len(values) != 1: + raise ValueError("scalar projection must contain exactly one value") + transforms = tuple(self.transforms) + if any(not isinstance(transform, str) or not transform.strip() for transform in transforms): + raise ValueError("projection transforms must be non-empty strings") + transforms = tuple(transform.strip() for transform in transforms) + if len(transforms) != len(set(transforms)): + raise ValueError("projection transforms must be unique") + object.__setattr__(self, "argument", self.argument.strip()) + object.__setattr__(self, "group", self.group.strip()) + object.__setattr__(self, "values", values) + object.__setattr__(self, "transforms", transforms) + + +@dataclass(frozen=True) +class UserConfigOverride: + """Explicit user-owned final value for one projected configuration argument.""" + + argument: str + value: object + source_path: str + + def __post_init__(self): + """Detach the override value and retain its explicit source path.""" + if not isinstance(self.argument, str) or not self.argument.strip(): + raise ValueError("override argument must be a non-empty string") + if not isinstance(self.source_path, str) or not self.source_path.strip(): + raise ValueError("override source_path must be a non-empty string") + value = _freeze(copy.deepcopy(self.value)) + object.__setattr__(self, "argument", self.argument.strip()) + object.__setattr__(self, "value", value) + object.__setattr__(self, "source_path", self.source_path.strip()) + + @dataclass(frozen=True) class ProviderSnapshot: """One integration's immutable generation of health, topology, and aliases.""" @@ -132,6 +284,7 @@ class ProviderSnapshot: aliases: tuple = () identity_aliases: tuple = () role_assignments: tuple = () + config_projections: tuple = () def __post_init__(self): """Validate scalar fields and detach caller-owned mutable data.""" @@ -152,11 +305,15 @@ def __post_init__(self): role_assignments = tuple(self.role_assignments) if any(not isinstance(assignment, ProviderRoleAssignment) for assignment in role_assignments): raise ValueError("role_assignments must contain ProviderRoleAssignment values") + config_projections = tuple(self.config_projections) + if any(not isinstance(projection, ProviderConfigProjection) for projection in config_projections): + raise ValueError("config_projections must contain ProviderConfigProjection values") object.__setattr__(self, "provider_id", self.provider_id.strip()) object.__setattr__(self, "topology_fragment", _freeze(copy.deepcopy(dict(self.topology_fragment)))) object.__setattr__(self, "aliases", aliases) object.__setattr__(self, "identity_aliases", identity_aliases) object.__setattr__(self, "role_assignments", role_assignments) + object.__setattr__(self, "config_projections", config_projections) @dataclass(frozen=True) @@ -253,6 +410,43 @@ class AutoConfigField: provenance: tuple +@dataclass(frozen=True) +class ProjectionCandidate: + """One normalized provider candidate for a projected argument.""" + + argument: str + provider_id: str + generation: int + role: str + group: str + routing: str + cardinality: str + required: bool + values: tuple + value_kinds: tuple + capabilities: tuple + identity_selectors: tuple + access_path_ids: tuple + transforms: tuple + specificities: tuple + provenance: tuple + + +@dataclass(frozen=True) +class ProjectedConfigArgument: + """Effective immutable PredBat argument plus all provider candidates.""" + + name: str + value: object + cardinality: str + required: bool + transforms: tuple + provenance: tuple + candidate_provenance: tuple + candidates: tuple + override_source: Optional[str] = None + + @dataclass(frozen=True) class AutoConfigPlan: """Deterministic immutable result of compiling all usable fragments.""" @@ -267,6 +461,8 @@ class AutoConfigPlan: primary_target: Optional[str] control_target: Optional[str] materialization_readiness: MaterializationReadiness + config_arguments: tuple + projected_config: Mapping fields: tuple provenance: tuple provider_generations: tuple @@ -350,6 +546,31 @@ def _semantic_topology(site): return topology +def _projection_payload(projection): + """Return one provider projection in deterministic plain form.""" + return { + "argument": projection.argument, + "role": projection.role.value, + "group": projection.group, + "routing": projection.routing.value, + "cardinality": projection.cardinality.value, + "required": projection.required, + "transforms": projection.transforms, + "values": [ + { + "node_id": value.node_id, + "kind": value.kind.value, + "value": value.value, + "capability": value.capability, + "identity_kind": value.identity_kind, + "identity_value": value.identity_value, + "access_path_id": value.access_path_id, + } + for value in projection.values + ], + } + + def _fingerprint_snapshot(snapshot): """Bind a provider generation to exactly one health/fragment/alias value.""" aliases = [ @@ -377,6 +598,19 @@ def _fingerprint_snapshot(snapshot): } for assignment in sorted(snapshot.role_assignments, key=lambda item: (item.group, item.role.value, item.index, item.node_id)) ] + config_projections = [ + _projection_payload(projection) + for projection in sorted( + snapshot.config_projections, + key=lambda item: ( + item.argument, + item.group, + item.role.value, + item.routing.value, + item.cardinality.value, + ), + ) + ] payload = { "provider": snapshot.provider_id, "generation": snapshot.generation, @@ -385,6 +619,7 @@ def _fingerprint_snapshot(snapshot): "aliases": aliases, "identity_aliases": identity_aliases, "role_assignments": role_assignments, + "config_projections": config_projections, } return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() @@ -682,6 +917,359 @@ def _compile_roles(snapshots, bindings, identity_bindings, provider_nodes, canon return role_bindings, primary_targets, control_targets, legacy_targets, readiness +def _projection_sort_key(projection): + """Return the stable ordering key for provider projection declarations.""" + return ( + projection.argument, + projection.group, + projection.role.value, + projection.routing.value, + projection.cardinality.value, + _canonical_json(_projection_payload(projection)), + ) + + +def _projection_topology_sources(document, provider_id): + """Index provider-owned access paths and capability-to-path bindings.""" + access_paths = {} + capabilities = set() + for node in document.get("nodes", ()): + node_id = str(node["id"]) + for access_path in node.get("accessPaths", ()): + path_id = str(access_path["id"]) + owner = access_path.get("provider", provider_id) + access_paths[(node_id, path_id)] = owner + for capability in node.get("capabilities", ()): + capabilities.add( + ( + node_id, + str(capability["capability"]), + str(capability["accessPath"]), + ) + ) + return access_paths, capabilities + + +def _projection_override_value(override, candidate): + """Validate and return an override in the provider candidate's shape.""" + value = override.value + if candidate.cardinality == ProjectionCardinality.PER_INDEX.value: + if not isinstance(value, tuple): + raise AutoConfigCompileError("override {} must be an ordered per-index sequence".format(override.argument)) + if len(value) != len(candidate.values): + raise AutoConfigCompileError( + "override {} cardinality mismatch: expected {}, got {}".format( + override.argument, + len(candidate.values), + len(value), + ) + ) + values = value + else: + if isinstance(value, (tuple, Mapping)): + raise AutoConfigCompileError("override {} must be a scalar value".format(override.argument)) + values = (value,) + for item in values: + if not isinstance(item, (str, int, float, bool)) and item is not None: + raise AutoConfigCompileError("override {} contains a non-scalar value".format(override.argument)) + _canonical_json(item) + if candidate.required and any(item is None for item in values): + raise AutoConfigCompileError("override {} leaves a required slot empty".format(override.argument)) + return value + + +def _compile_config_projections( + snapshots, + primary_targets, + control_targets, + provider_nodes, + canonical_by_node, + documents_by_provider, + user_overrides, +): + """Compile provider projection candidates into effective PredBat arguments.""" + targets_by_role_group = {} + for role, targets in ( + (AliasRole.PRIMARY, primary_targets), + (AliasRole.CONTROL, control_targets), + ): + for target in targets: + targets_by_role_group.setdefault((role, target.group), []).append(target) + targets_by_role_group = {key: tuple(sorted(targets, key=lambda item: item.index)) for key, targets in targets_by_role_group.items()} + + candidates_by_argument = {} + for snapshot in snapshots: + access_paths, capabilities = _projection_topology_sources( + documents_by_provider[snapshot.provider_id], + snapshot.provider_id, + ) + identity_assertions = {(alias.kind, alias.value, alias.node_id) for alias in snapshot.identity_aliases} + seen_arguments = set() + for projection_index, projection in enumerate(sorted(snapshot.config_projections, key=_projection_sort_key)): + if projection.argument in seen_arguments: + raise AutoConfigCompileError( + "provider {} repeats projection argument {}".format( + snapshot.provider_id, + projection.argument, + ) + ) + seen_arguments.add(projection.argument) + targets = targets_by_role_group.get( + (projection.role, projection.group), + (), + ) + if not targets: + raise AutoConfigCompileError( + "projection {} has no selected {} targets in group {}".format( + projection.argument, + projection.role.value, + projection.group, + ) + ) + if projection.cardinality is ProjectionCardinality.PER_INDEX and len(projection.values) != len(targets): + raise AutoConfigCompileError( + "projection {} cardinality mismatch: expected {}, got {}".format( + projection.argument, + len(targets), + len(projection.values), + ) + ) + if projection.routing is ProjectionRouting.COORDINATOR and len({target.node_id for target in targets}) != 1: + raise AutoConfigCompileError("coordinator projection {} requires one canonical target".format(projection.argument)) + + values = [] + kinds = [] + specificities = [] + provenance = [] + for value_index, value in enumerate(projection.values): + if value.node_id not in provider_nodes[snapshot.provider_id]: + raise AutoConfigCompileError( + "projection {} targets unknown provider-local node {}".format( + projection.argument, + value.node_id, + ) + ) + if value.kind is ProjectionValueKind.ENTITY: + matching_capabilities = { + access_path_id + for ( + node_id, + capability, + access_path_id, + ) in capabilities + if node_id == value.node_id and capability == value.capability + } + if not matching_capabilities: + raise AutoConfigCompileError( + "projection {} capability {} is not published by node {}".format( + projection.argument, + value.capability, + value.node_id, + ) + ) + if value.access_path_id is not None and value.access_path_id not in matching_capabilities: + raise AutoConfigCompileError( + "projection {} capability {} is not bound to access path {}".format( + projection.argument, + value.capability, + value.access_path_id, + ) + ) + target_index = value_index if projection.cardinality is ProjectionCardinality.PER_INDEX else 0 + canonical_node_id = canonical_by_node[(snapshot.provider_id, value.node_id)] + target = targets[target_index] + if canonical_node_id != target.node_id: + raise AutoConfigCompileError( + "projection {} slot {} is unrelated to selected canonical target {}".format( + projection.argument, + target_index, + target.node_id, + ) + ) + + specificity = 0 + if value.identity_kind is not None: + identity = ( + value.identity_kind, + value.identity_value, + value.node_id, + ) + if identity not in identity_assertions: + raise AutoConfigCompileError( + "projection {} identity selector is not asserted by provider {}".format( + projection.argument, + snapshot.provider_id, + ) + ) + specificity = 1 + if value.access_path_id is not None: + owner = access_paths.get((value.node_id, value.access_path_id)) + if owner != snapshot.provider_id: + raise AutoConfigCompileError( + "projection {} access path {} is not provider-owned".format( + projection.argument, + value.access_path_id, + ) + ) + specificity = 2 + if projection.required and value.kind is ProjectionValueKind.NONE: + raise AutoConfigCompileError( + "projection {} leaves required slot {} empty".format( + projection.argument, + target_index, + ) + ) + + field_path = "/projected_config/{}".format(projection.argument) + if projection.cardinality is ProjectionCardinality.PER_INDEX: + field_path += "/{}".format(target_index) + values.append(value.value) + kinds.append(value.kind.value) + specificities.append(specificity) + provenance.append( + ( + FieldProvenance( + field_path, + snapshot.provider_id, + snapshot.generation, + "/config_projections/{}/values/{}".format( + projection_index, + value_index, + ), + ), + ) + ) + + candidate = ProjectionCandidate( + argument=projection.argument, + provider_id=snapshot.provider_id, + generation=snapshot.generation, + role=projection.role.value, + group=projection.group, + routing=projection.routing.value, + cardinality=projection.cardinality.value, + required=projection.required, + values=tuple(values), + value_kinds=tuple(kinds), + capabilities=tuple(value.capability for value in projection.values), + identity_selectors=tuple( + ( + value.identity_kind, + value.identity_value, + ) + if value.identity_kind is not None + else None + for value in projection.values + ), + access_path_ids=tuple(value.access_path_id for value in projection.values), + transforms=projection.transforms, + specificities=tuple(specificities), + provenance=tuple(provenance), + ) + candidates_by_argument.setdefault( + projection.argument, + [], + ).append(candidate) + + overrides = {} + for override in tuple(user_overrides): + if not isinstance(override, UserConfigOverride): + raise ValueError("user_overrides must contain UserConfigOverride values") + if override.argument in overrides: + raise AutoConfigCompileError("duplicate user override for {}".format(override.argument)) + overrides[override.argument] = override + unknown_overrides = sorted(set(overrides) - set(candidates_by_argument)) + if unknown_overrides: + raise AutoConfigCompileError("user override has no provider candidate: {}".format(", ".join(unknown_overrides))) + + arguments = [] + projected_config = {} + for argument, unordered_candidates in sorted(candidates_by_argument.items()): + candidates = tuple( + sorted( + unordered_candidates, + key=lambda item: (item.provider_id, item.generation), + ) + ) + metadata = { + ( + candidate.role, + candidate.group, + candidate.routing, + candidate.cardinality, + candidate.required, + candidate.transforms, + len(candidate.values), + ) + for candidate in candidates + } + if len(metadata) != 1: + raise AutoConfigCompileError("conflicting projection contract for {}".format(argument)) + first = candidates[0] + candidate_provenance = tuple(source for candidate in candidates for slot_sources in candidate.provenance for source in slot_sources) + override = overrides.get(argument) + selected_values = [] + selected_provenance = [] + if override is None: + for index in range(len(first.values)): + highest_specificity = max(candidate.specificities[index] for candidate in candidates) + selected = tuple(candidate for candidate in candidates if candidate.specificities[index] == highest_specificity) + kinds = {candidate.value_kinds[index] for candidate in selected} + if len(kinds) != 1: + raise AutoConfigCompileError( + "projection {} type mismatch at slot {}".format( + argument, + index, + ) + ) + canonical_values = {_canonical_json(candidate.values[index]) for candidate in selected} + if len(canonical_values) != 1: + raise AutoConfigCompileError( + "ambiguous multi-provider projection {} at slot {}".format( + argument, + index, + ) + ) + selected_values.append(selected[0].values[index]) + selected_provenance.extend(source for candidate in selected for source in candidate.provenance[index]) + if first.cardinality == ProjectionCardinality.PER_INDEX.value: + effective_value = tuple(selected_values) + else: + effective_value = selected_values[0] + override_source = None + else: + effective_value = _projection_override_value(override, first) + override_source = override.source_path + selected_provenance = [ + FieldProvenance( + "/projected_config/{}".format(argument), + "user_override", + 0, + override.source_path, + ) + ] + + effective_value = _freeze(effective_value) + projected_config[argument] = effective_value + arguments.append( + ProjectedConfigArgument( + name=argument, + value=effective_value, + cardinality=first.cardinality, + required=first.required, + transforms=first.transforms, + provenance=tuple(selected_provenance), + candidate_provenance=candidate_provenance, + candidates=candidates, + override_source=override_source, + ) + ) + return ( + tuple(arguments), + _freeze(projected_config), + ) + + def _field_provenance( snapshots, bindings, @@ -745,7 +1333,7 @@ def _field_provenance( return tuple(sorted(provenance, key=lambda item: (item.field_path, item.provider_id, item.generation, item.source_path))) -def compile_auto_config(snapshots): +def compile_auto_config(snapshots, user_overrides=()): """Compile usable provider snapshots into one deterministic immutable plan.""" snapshots = tuple(sorted(snapshots, key=lambda item: item.provider_id)) if not snapshots: @@ -755,6 +1343,7 @@ def compile_auto_config(snapshots): raise AutoConfigCompileError("duplicate provider snapshots are not allowed") documents = [] + documents_by_provider = {} provider_nodes = {} for snapshot in snapshots: document = decode_topology(_plain(snapshot.topology_fragment)) @@ -765,6 +1354,7 @@ def compile_auto_config(snapshots): raise AutoConfigCompileError("provider {} supplied fragment owned by {}".format(snapshot.provider_id, document_provider)) nodes = _document_node_ids(document, snapshot.provider_id) provider_nodes[snapshot.provider_id] = nodes + documents_by_provider[snapshot.provider_id] = document documents.append(document) documents, canonical_by_node, identity_bindings = _correlate_identities(snapshots, documents, provider_nodes) @@ -800,6 +1390,22 @@ def compile_auto_config(snapshots): control_target = legacy_targets[AliasRole.CONTROL] topology_snapshot = merge_topologies(documents) + config_arguments, projected_config = _compile_config_projections( + snapshots, + primary_targets, + control_targets, + provider_nodes, + canonical_by_node, + documents_by_provider, + user_overrides, + ) + if config_arguments: + blockers = tuple(blocker for blocker in readiness.blockers if blocker != "config_projection_bindings_missing") + ("atomic_materializer_missing",) + readiness = MaterializationReadiness( + ready=False, + blockers=blockers, + ) + fields = [] for binding in bindings: source = FieldProvenance( @@ -835,6 +1441,14 @@ def compile_auto_config(snapshots): target.provenance, ) ) + for argument in config_arguments: + fields.append( + AutoConfigField( + "config.{}".format(argument.name), + argument.value, + argument.provenance, + ) + ) fields = tuple(sorted(fields, key=lambda item: item.name)) semantic = { @@ -876,10 +1490,21 @@ def compile_auto_config(snapshots): "ready": readiness.ready, "blockers": readiness.blockers, }, + "config_arguments": [ + { + "name": argument.name, + "value": argument.value, + "cardinality": argument.cardinality, + "required": argument.required, + "transforms": argument.transforms, + } + for argument in config_arguments + ], + "projected_config": projected_config, "fields": [{"name": field.name, "value": field.value} for field in fields], } digest = hashlib.sha256(_canonical_json(semantic).encode("utf-8")).hexdigest() - provenance = _field_provenance( + base_provenance = _field_provenance( snapshots, bindings, identity_bindings, @@ -890,6 +1515,18 @@ def compile_auto_config(snapshots): topology_snapshot, canonical_by_node, ) + projection_provenance = tuple(source for argument in config_arguments for source in (argument.candidate_provenance + argument.provenance)) + provenance = tuple( + sorted( + set(base_provenance + projection_provenance), + key=lambda item: ( + item.field_path, + item.provider_id, + item.generation, + item.source_path, + ), + ) + ) return AutoConfigPlan( digest=digest, topology=_freeze(topology_snapshot.site), @@ -901,6 +1538,8 @@ def compile_auto_config(snapshots): primary_target=primary_target, control_target=control_target, materialization_readiness=readiness, + config_arguments=config_arguments, + projected_config=projected_config, fields=fields, provenance=provenance, provider_generations=tuple((snapshot.provider_id, snapshot.generation) for snapshot in snapshots), diff --git a/apps/predbat/tests/test_lattice_autoconfig.py b/apps/predbat/tests/test_lattice_autoconfig.py index 285fb4010..edc4965fc 100644 --- a/apps/predbat/tests/test_lattice_autoconfig.py +++ b/apps/predbat/tests/test_lattice_autoconfig.py @@ -16,11 +16,17 @@ CompileStatus, LatticeAutoConfigCompiler, MaterializationReadiness, + ProjectionCardinality, + ProjectionRouting, + ProjectionValueKind, + ProviderConfigProjection, ProviderAlias, ProviderHealth, ProviderIdentityAlias, + ProviderProjectionValue, ProviderRoleAssignment, ProviderSnapshot, + UserConfigOverride, compile_auto_config, ) @@ -75,6 +81,118 @@ def snapshot( ) +def multi_fragment(provider, node_ids): + """Build a provider fragment containing ordered independent inverter nodes.""" + nodes = [] + for index, node_id in enumerate(node_ids): + access_path = "{}-{}-path".format(provider, node_id) + nodes.append( + { + "id": node_id, + "kind": "inverter", + "deviceType": "hybrid", + "accessPaths": [ + { + "id": access_path, + "provider": provider, + "preference": 10, + } + ], + "capabilities": [ + { + "capability": "battery.target_soc", + "accessPath": access_path, + "ref": index + 1, + "shape": "setpoint", + "control": {"protocol": "mqtt"}, + } + ], + } + ) + return { + "topologyVersion": "0.3.0", + "scope": "fragment", + "docVersion": 1, + "producer": { + "name": provider, + "provider": provider, + "authority": 10, + }, + "nodes": nodes, + } + + +def projection_snapshot( + provider, + node_ids, + role_assignments, + config_projections, + identity_aliases=(), + generation=1, +): + """Build a provider snapshot carrying indexed roles and config projections.""" + return ProviderSnapshot( + provider_id=provider, + generation=generation, + health=ProviderHealth.HEALTHY, + topology_fragment=multi_fragment(provider, node_ids), + identity_aliases=identity_aliases, + role_assignments=role_assignments, + config_projections=config_projections, + ) + + +def projection_value( + node_id, + kind, + value=None, + capability=None, + identity=None, + access_path_id=None, +): + """Build one compact provider projection value for tests.""" + identity_kind, identity_value = identity or (None, None) + if kind is ProjectionValueKind.ENTITY and capability is None: + capability = "battery.target_soc" + return ProviderProjectionValue( + node_id=node_id, + kind=kind, + value=value, + capability=capability, + identity_kind=identity_kind, + identity_value=identity_value, + access_path_id=access_path_id, + ) + + +def config_projection( + argument, + values, + role=AliasRole.PRIMARY, + group="inverters", + routing=ProjectionRouting.LEAF, + cardinality=ProjectionCardinality.PER_INDEX, + required=True, + transforms=(), +): + """Build one generic provider projection declaration for tests.""" + return ProviderConfigProjection( + argument=argument, + role=role, + group=group, + routing=routing, + cardinality=cardinality, + values=values, + required=required, + transforms=transforms, + ) + + +def indexed_roles(node_ids): + """Select each node as both an indexed primary and control target.""" + return tuple(ProviderRoleAssignment(role, "inverters", index, node_id) for index, node_id in enumerate(node_ids) for role in (AliasRole.PRIMARY, AliasRole.CONTROL)) + + def ready_plan(): """Copy a compiled shadow plan into a future-materializer test harness.""" plan = compile_auto_config((snapshot("gateway"),)) @@ -403,6 +521,658 @@ def test_reference_only_plan_is_explicitly_shadow_only(self): self.assertIsNone(plan.control_target) +class TestConfigProjectionCompilation(unittest.TestCase): + """Provider contracts project selected indexed capabilities into config.""" + + def test_gateway_multi_aio_projects_ordered_leaf_arrays(self): + """Two Gateway-like AIO nodes produce deterministic per-index arrays.""" + nodes = ("AIO1", "AIO2") + gateway = projection_snapshot( + "gateway", + nodes, + indexed_roles(nodes), + ( + config_projection( + "battery_power", + tuple( + projection_value( + node, + ProjectionValueKind.ENTITY, + "sensor.gateway_{}_battery_power".format(node.lower()), + ) + for node in nodes + ), + ), + config_projection( + "inverter_type", + tuple( + projection_value( + node, + ProjectionValueKind.CONSTANT, + "GEC", + ) + for node in nodes + ), + ), + config_projection( + "num_inverters", + ( + projection_value( + nodes[0], + ProjectionValueKind.CONSTANT, + 2, + ), + ), + cardinality=ProjectionCardinality.SCALAR, + ), + ), + ) + + plan = compile_auto_config((gateway,)) + + self.assertEqual( + dict(plan.projected_config), + { + "battery_power": ( + "sensor.gateway_aio1_battery_power", + "sensor.gateway_aio2_battery_power", + ), + "inverter_type": ("GEC", "GEC"), + "num_inverters": 2, + }, + ) + self.assertEqual( + [argument.name for argument in plan.config_arguments], + ["battery_power", "inverter_type", "num_inverters"], + ) + self.assertEqual( + {field.name for field in plan.fields if field.name.startswith("config.")}, + { + "config.battery_power", + "config.inverter_type", + "config.num_inverters", + }, + ) + self.assertEqual( + plan.materialization_readiness.blockers, + ("atomic_materializer_missing",), + ) + self.assertFalse(plan.materialization_readiness.ready) + requests = [] + run = LatticeAutoConfigCompiler( + {"gateway": MutableReader(gateway)}, + requests.append, + ).drain() + self.assertEqual(run.materializations, 0) + self.assertEqual(requests, []) + + def test_ge_ems_coordinator_fans_out_entities_and_zero_constants(self): + """One selected EMS coordinator can publish an ordered fan-out array.""" + role_assignments = ( + ProviderRoleAssignment( + AliasRole.PRIMARY, + "inverters", + 0, + "BAT1", + ), + ProviderRoleAssignment( + AliasRole.PRIMARY, + "inverters", + 1, + "BAT2", + ), + ProviderRoleAssignment( + AliasRole.CONTROL, + "inverters", + 0, + "EMS", + ), + ProviderRoleAssignment( + AliasRole.CONTROL, + "inverters", + 1, + "EMS", + ), + ) + ems = projection_snapshot( + "gecloud", + ("EMS", "BAT1", "BAT2"), + role_assignments, + ( + config_projection( + "battery_power", + ( + projection_value( + "EMS", + ProjectionValueKind.ENTITY, + "sensor.gecloud_ems_battery_power", + ), + projection_value( + "EMS", + ProjectionValueKind.CONSTANT, + 0, + ), + ), + role=AliasRole.CONTROL, + routing=ProjectionRouting.COORDINATOR, + ), + config_projection( + "charge_start_time", + ( + projection_value( + "EMS", + ProjectionValueKind.ENTITY, + "select.gecloud_ems_charge_start", + ), + projection_value( + "EMS", + ProjectionValueKind.ENTITY, + "select.gecloud_ems_charge_start", + ), + ), + role=AliasRole.CONTROL, + routing=ProjectionRouting.COORDINATOR, + ), + ), + ) + + plan = compile_auto_config((ems,)) + + self.assertEqual( + plan.projected_config["battery_power"], + ("sensor.gecloud_ems_battery_power", 0), + ) + self.assertEqual( + plan.projected_config["charge_start_time"], + ( + "select.gecloud_ems_charge_start", + "select.gecloud_ems_charge_start", + ), + ) + battery_argument = next(argument for argument in plan.config_arguments if argument.name == "battery_power") + self.assertEqual( + battery_argument.candidates[0].routing, + ProjectionRouting.COORDINATOR.value, + ) + + def test_fox_projection_carries_transform_and_constant_flags(self): + """Fox-like power entities retain transform metadata and invert flags.""" + fox = projection_snapshot( + "fox", + ("FOX1",), + indexed_roles(("FOX1",)), + ( + config_projection( + "grid_power", + ( + projection_value( + "FOX1", + ProjectionValueKind.ENTITY, + "sensor.fox_fox1_grid_power", + ), + ), + transforms=("invert", "watts"), + ), + config_projection( + "grid_power_invert", + ( + projection_value( + "FOX1", + ProjectionValueKind.CONSTANT, + True, + ), + ), + ), + config_projection( + "inverter_type", + ( + projection_value( + "FOX1", + ProjectionValueKind.CONSTANT, + "FoxCloud", + ), + ), + ), + ), + ) + + plan = compile_auto_config((fox,)) + grid_argument = next(argument for argument in plan.config_arguments if argument.name == "grid_power") + + self.assertEqual(grid_argument.transforms, ("invert", "watts")) + self.assertEqual(plan.projected_config["grid_power_invert"], (True,)) + self.assertEqual(plan.projected_config["inverter_type"], ("FoxCloud",)) + + def test_solis_optional_projection_preserves_none_and_absence(self): + """Solis-like optional args distinguish explicit None from no binding.""" + solis = projection_snapshot( + "solis", + ("SOLIS1",), + indexed_roles(("SOLIS1",)), + ( + config_projection( + "givtcp_rest", + ( + projection_value( + "SOLIS1", + ProjectionValueKind.NONE, + ), + ), + cardinality=ProjectionCardinality.SCALAR, + required=False, + ), + config_projection( + "pause_mode", + ( + projection_value( + "SOLIS1", + ProjectionValueKind.NONE, + ), + ), + required=False, + ), + config_projection( + "inverter_type", + ( + projection_value( + "SOLIS1", + ProjectionValueKind.CONSTANT, + "SolisCloud", + ), + ), + ), + ), + ) + + plan = compile_auto_config((solis,)) + + self.assertIsNone(plan.projected_config["givtcp_rest"]) + self.assertEqual(plan.projected_config["pause_mode"], (None,)) + self.assertNotIn("idle_start_time", plan.projected_config) + + def test_cross_provider_identity_and_access_path_select_one_value(self): + """An explicit correlated access path beats a generic provider value.""" + gateway = projection_snapshot( + "gateway", + ("GW1",), + indexed_roles(("GW1",)), + ( + config_projection( + "battery_power", + ( + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway_battery_power", + ), + ), + ), + ), + identity_aliases=(ProviderIdentityAlias("serial", "SER123", "GW1"),), + ) + cloud = projection_snapshot( + "cloud", + ("CLOUD1",), + (), + ( + config_projection( + "battery_power", + ( + projection_value( + "CLOUD1", + ProjectionValueKind.ENTITY, + "sensor.cloud_battery_power", + identity=("serial", "SER123"), + access_path_id="cloud-CLOUD1-path", + ), + ), + ), + ), + identity_aliases=( + ProviderIdentityAlias( + "serial", + "SER123", + "CLOUD1", + ), + ), + ) + + plan = compile_auto_config((gateway, cloud)) + argument = plan.config_arguments[0] + + self.assertEqual( + plan.projected_config["battery_power"], + ("sensor.cloud_battery_power",), + ) + self.assertEqual( + {candidate.provider_id for candidate in argument.candidates}, + {"gateway", "cloud"}, + ) + cloud_candidate = next(candidate for candidate in argument.candidates if candidate.provider_id == "cloud") + self.assertEqual( + cloud_candidate.capabilities, + ("battery.target_soc",), + ) + self.assertEqual( + cloud_candidate.identity_selectors, + (("serial", "SER123"),), + ) + self.assertEqual( + cloud_candidate.access_path_ids, + ("cloud-CLOUD1-path",), + ) + self.assertEqual( + {source.provider_id for source in argument.candidate_provenance}, + {"gateway", "cloud"}, + ) + + def test_user_override_wins_and_retains_provider_candidates(self): + """An explicit override resolves values without erasing candidates.""" + identity = "SER-OVERRIDE" + gateway = projection_snapshot( + "gateway", + ("GW1",), + indexed_roles(("GW1",)), + ( + config_projection( + "battery_power", + ( + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway_candidate", + ), + ), + ), + ), + identity_aliases=(ProviderIdentityAlias("serial", identity, "GW1"),), + ) + cloud = projection_snapshot( + "cloud", + ("CLOUD1",), + (), + ( + config_projection( + "battery_power", + ( + projection_value( + "CLOUD1", + ProjectionValueKind.ENTITY, + "sensor.cloud_candidate", + ), + ), + ), + ), + identity_aliases=(ProviderIdentityAlias("serial", identity, "CLOUD1"),), + ) + override = UserConfigOverride( + "battery_power", + ["sensor.user_selected"], + "/apps.yaml/battery_power", + ) + + plan = compile_auto_config( + (gateway, cloud), + user_overrides=(override,), + ) + argument = plan.config_arguments[0] + + self.assertEqual( + plan.projected_config["battery_power"], + ("sensor.user_selected",), + ) + self.assertEqual(argument.override_source, "/apps.yaml/battery_power") + self.assertEqual( + {source.provider_id for source in argument.candidate_provenance}, + {"gateway", "cloud"}, + ) + self.assertEqual( + [source.provider_id for source in argument.provenance], + ["user_override"], + ) + + def test_projection_order_and_outputs_are_deterministic_and_immutable(self): + """Provider and declaration order cannot alter or mutate the plan.""" + projections = ( + config_projection( + "inverter_type", + ( + projection_value( + "INV1", + ProjectionValueKind.CONSTANT, + "GEC", + ), + ), + ), + config_projection( + "battery_power", + ( + projection_value( + "INV1", + ProjectionValueKind.ENTITY, + "sensor.inv1_battery_power", + ), + ), + ), + ) + left_snapshot = projection_snapshot( + "gateway", + ("INV1",), + indexed_roles(("INV1",)), + projections, + ) + right_snapshot = projection_snapshot( + "gateway", + ("INV1",), + indexed_roles(("INV1",)), + tuple(reversed(projections)), + ) + + left = compile_auto_config((left_snapshot,)) + right = compile_auto_config((right_snapshot,)) + + self.assertEqual(left.digest, right.digest) + self.assertEqual( + [argument.name for argument in left.config_arguments], + ["battery_power", "inverter_type"], + ) + with self.assertRaises(TypeError): + left.projected_config["battery_power"] = ("changed",) + with self.assertRaises(TypeError): + left.projected_config["battery_power"][0] = "changed" + + def test_projection_invalidation_recompiles_a_new_provider_generation(self): + """Any provider can invalidate and replace its projection generation.""" + + def projected(generation, entity_id): + """Build one generation of the provider's projected entity.""" + return projection_snapshot( + "gateway", + ("INV1",), + indexed_roles(("INV1",)), + ( + config_projection( + "battery_power", + ( + projection_value( + "INV1", + ProjectionValueKind.ENTITY, + entity_id, + ), + ), + ), + ), + generation=generation, + ) + + reader = MutableReader(projected(1, "sensor.gateway_battery_power_v1")) + compiler = LatticeAutoConfigCompiler({"gateway": reader}) + first = compiler.drain() + + reader.value = projected(2, "sensor.gateway_battery_power_v2") + self.assertTrue( + compiler.invalidate( + "gateway", + 2, + "projection binding changed", + ) + ) + second = compiler.drain() + + self.assertNotEqual(first.plan.digest, second.plan.digest) + self.assertEqual( + second.plan.projected_config["battery_power"], + ("sensor.gateway_battery_power_v2",), + ) + self.assertEqual( + dict(second.plan.provider_generations), + {"gateway": 2}, + ) + + def test_projection_gaps_types_conflicts_and_unrelated_nodes_fail_closed(self): + """Unsafe required gaps, shapes, sources, and assertions are rejected.""" + roles = indexed_roles(("INV1", "INV2")) + required_gap = projection_snapshot( + "gateway", + ("INV1", "INV2"), + roles, + ( + config_projection( + "battery_power", + ( + projection_value( + "INV1", + ProjectionValueKind.ENTITY, + "sensor.inv1", + ), + projection_value( + "INV2", + ProjectionValueKind.NONE, + ), + ), + ), + ), + ) + with self.assertRaisesRegex( + AutoConfigCompileError, + "required slot", + ): + compile_auto_config((required_gap,)) + + wrong_cardinality = projection_snapshot( + "gateway", + ("INV1", "INV2"), + roles, + ( + config_projection( + "battery_power", + ( + projection_value( + "INV1", + ProjectionValueKind.ENTITY, + "sensor.inv1", + ), + ), + ), + ), + ) + with self.assertRaisesRegex( + AutoConfigCompileError, + "cardinality mismatch", + ): + compile_auto_config((wrong_cardinality,)) + + unrelated = projection_snapshot( + "gateway", + ("INV1", "OTHER"), + indexed_roles(("INV1",)), + ( + config_projection( + "battery_power", + ( + projection_value( + "OTHER", + ProjectionValueKind.ENTITY, + "sensor.other", + ), + ), + ), + ), + ) + with self.assertRaisesRegex( + AutoConfigCompileError, + "unrelated", + ): + compile_auto_config((unrelated,)) + + identity = "SER-CONFLICT" + gateway = projection_snapshot( + "gateway", + ("GW1",), + indexed_roles(("GW1",)), + ( + config_projection( + "battery_power", + ( + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway", + ), + ), + ), + ), + identity_aliases=(ProviderIdentityAlias("serial", identity, "GW1"),), + ) + cloud = projection_snapshot( + "cloud", + ("CLOUD1",), + (), + ( + config_projection( + "battery_power", + ( + projection_value( + "CLOUD1", + ProjectionValueKind.ENTITY, + "sensor.cloud", + ), + ), + ), + ), + identity_aliases=(ProviderIdentityAlias("serial", identity, "CLOUD1"),), + ) + with self.assertRaisesRegex( + AutoConfigCompileError, + "ambiguous multi-provider", + ): + compile_auto_config((gateway, cloud)) + + cloud_constant = projection_snapshot( + "cloud", + ("CLOUD1",), + (), + ( + config_projection( + "battery_power", + ( + projection_value( + "CLOUD1", + ProjectionValueKind.CONSTANT, + 0, + ), + ), + ), + ), + identity_aliases=(ProviderIdentityAlias("serial", identity, "CLOUD1"),), + ) + with self.assertRaisesRegex( + AutoConfigCompileError, + "type mismatch", + ): + compile_auto_config((gateway, cloud_constant)) + + class TestInvalidationStateMachine(unittest.TestCase): """Invalidations coalesce without losing freshness or last-known-good state.""" From 47f263eb3f006770fecf0a839b18b01d8388741c Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Tue, 28 Jul 2026 06:52:00 +0100 Subject: [PATCH 4/8] feat(lattice): compose partial provider projections --- apps/predbat/lattice_autoconfig.py | 188 +++++++-- apps/predbat/tests/test_lattice_autoconfig.py | 390 ++++++++++++++++++ 2 files changed, 542 insertions(+), 36 deletions(-) diff --git a/apps/predbat/lattice_autoconfig.py b/apps/predbat/lattice_autoconfig.py index 42a24c957..f89f9a928 100644 --- a/apps/predbat/lattice_autoconfig.py +++ b/apps/predbat/lattice_autoconfig.py @@ -11,7 +11,7 @@ gated runtime component can supply a materializer. """ -# cspell:ignore autoconfig +# cspell:ignore autoconfig popleft import copy import hashlib @@ -423,6 +423,8 @@ class ProjectionCandidate: cardinality: str required: bool values: tuple + slot_indexes: tuple + target_count: int value_kinds: tuple capabilities: tuple identity_selectors: tuple @@ -956,11 +958,11 @@ def _projection_override_value(override, candidate): if candidate.cardinality == ProjectionCardinality.PER_INDEX.value: if not isinstance(value, tuple): raise AutoConfigCompileError("override {} must be an ordered per-index sequence".format(override.argument)) - if len(value) != len(candidate.values): + if len(value) != candidate.target_count: raise AutoConfigCompileError( "override {} cardinality mismatch: expected {}, got {}".format( override.argument, - len(candidate.values), + candidate.target_count, len(value), ) ) @@ -978,6 +980,111 @@ def _projection_override_value(override, candidate): return value +def _projection_slot_indexes(projection, snapshot, targets, canonical_by_node): + """Map provider-local projection values onto aggregate target slots.""" + if projection.cardinality is ProjectionCardinality.SCALAR: + canonical_node_id = canonical_by_node[ + ( + snapshot.provider_id, + projection.values[0].node_id, + ) + ] + target = targets[0] + if canonical_node_id != target.node_id: + raise AutoConfigCompileError( + "projection {} slot 0 is unrelated to selected canonical target {}".format( + projection.argument, + target.node_id, + ) + ) + return (0,) + + targets_by_index = {target.index: target for target in targets} + owned_by_local_node = {} + for assignment in sorted( + snapshot.role_assignments, + key=lambda item: ( + item.group, + item.role.value, + item.index, + item.node_id, + ), + ): + if assignment.role is not projection.role or assignment.group != projection.group: + continue + target = targets_by_index.get(assignment.index) + if target is None: + continue + canonical_node_id = canonical_by_node[ + ( + snapshot.provider_id, + assignment.node_id, + ) + ] + if target.node_id == canonical_node_id: + owned_by_local_node.setdefault( + assignment.node_id, + deque(), + ).append(assignment.index) + + available_by_node = {} + for target in targets: + available_by_node.setdefault(target.node_id, deque()).append(target.index) + unowned_value_counts = {} + for value in projection.values: + if value.node_id in owned_by_local_node: + continue + canonical_node_id = canonical_by_node[ + ( + snapshot.provider_id, + value.node_id, + ) + ] + unowned_value_counts[canonical_node_id] = unowned_value_counts.get(canonical_node_id, 0) + 1 + + slot_indexes = [] + used_slots = set() + for value in projection.values: + canonical_node_id = canonical_by_node[ + ( + snapshot.provider_id, + value.node_id, + ) + ] + available = owned_by_local_node.get(value.node_id) + if available is None: + available = available_by_node.get(canonical_node_id) + while available and available[0] in used_slots: + available.popleft() + if available and unowned_value_counts[canonical_node_id] != len(available): + raise AutoConfigCompileError( + "projection {} value for {} is ambiguous across aggregate slots {}".format( + projection.argument, + value.node_id, + tuple(available), + ) + ) + unowned_value_counts[canonical_node_id] -= 1 + if not available: + raise AutoConfigCompileError( + "projection {} value for {} is unrelated to an available selected target".format( + projection.argument, + value.node_id, + ) + ) + target_index = available.popleft() + if target_index in used_slots: + raise AutoConfigCompileError( + "projection {} repeats aggregate slot {}".format( + projection.argument, + target_index, + ) + ) + slot_indexes.append(target_index) + used_slots.add(target_index) + return tuple(slot_indexes) + + def _compile_config_projections( snapshots, primary_targets, @@ -1026,22 +1133,9 @@ def _compile_config_projections( projection.group, ) ) - if projection.cardinality is ProjectionCardinality.PER_INDEX and len(projection.values) != len(targets): - raise AutoConfigCompileError( - "projection {} cardinality mismatch: expected {}, got {}".format( - projection.argument, - len(targets), - len(projection.values), - ) - ) if projection.routing is ProjectionRouting.COORDINATOR and len({target.node_id for target in targets}) != 1: raise AutoConfigCompileError("coordinator projection {} requires one canonical target".format(projection.argument)) - - values = [] - kinds = [] - specificities = [] - provenance = [] - for value_index, value in enumerate(projection.values): + for value in projection.values: if value.node_id not in provider_nodes[snapshot.provider_id]: raise AutoConfigCompileError( "projection {} targets unknown provider-local node {}".format( @@ -1049,6 +1143,18 @@ def _compile_config_projections( value.node_id, ) ) + slot_indexes = _projection_slot_indexes( + projection, + snapshot, + targets, + canonical_by_node, + ) + + values = [] + kinds = [] + specificities = [] + provenance = [] + for value_index, value in enumerate(projection.values): if value.kind is ProjectionValueKind.ENTITY: matching_capabilities = { access_path_id @@ -1075,17 +1181,7 @@ def _compile_config_projections( value.access_path_id, ) ) - target_index = value_index if projection.cardinality is ProjectionCardinality.PER_INDEX else 0 - canonical_node_id = canonical_by_node[(snapshot.provider_id, value.node_id)] - target = targets[target_index] - if canonical_node_id != target.node_id: - raise AutoConfigCompileError( - "projection {} slot {} is unrelated to selected canonical target {}".format( - projection.argument, - target_index, - target.node_id, - ) - ) + target_index = slot_indexes[value_index] specificity = 0 if value.identity_kind is not None: @@ -1150,6 +1246,8 @@ def _compile_config_projections( cardinality=projection.cardinality.value, required=projection.required, values=tuple(values), + slot_indexes=slot_indexes, + target_count=(len(targets) if projection.cardinality is ProjectionCardinality.PER_INDEX else 1), value_kinds=tuple(kinds), capabilities=tuple(value.capability for value in projection.values), identity_selectors=tuple( @@ -1199,7 +1297,7 @@ def _compile_config_projections( candidate.cardinality, candidate.required, candidate.transforms, - len(candidate.values), + candidate.target_count, ) for candidate in candidates } @@ -1211,10 +1309,28 @@ def _compile_config_projections( selected_values = [] selected_provenance = [] if override is None: - for index in range(len(first.values)): - highest_specificity = max(candidate.specificities[index] for candidate in candidates) - selected = tuple(candidate for candidate in candidates if candidate.specificities[index] == highest_specificity) - kinds = {candidate.value_kinds[index] for candidate in selected} + for index in range(first.target_count): + slot_candidates = tuple( + ( + candidate, + candidate.slot_indexes.index(index), + ) + for candidate in candidates + if index in candidate.slot_indexes + ) + if not slot_candidates: + if first.required: + raise AutoConfigCompileError( + "projection {} cardinality mismatch: required slot {} has no provider candidate".format( + argument, + index, + ) + ) + selected_values.append(None) + continue + highest_specificity = max(candidate.specificities[value_index] for candidate, value_index in slot_candidates) + selected = tuple((candidate, value_index) for candidate, value_index in slot_candidates if candidate.specificities[value_index] == highest_specificity) + kinds = {candidate.value_kinds[value_index] for candidate, value_index in selected} if len(kinds) != 1: raise AutoConfigCompileError( "projection {} type mismatch at slot {}".format( @@ -1222,7 +1338,7 @@ def _compile_config_projections( index, ) ) - canonical_values = {_canonical_json(candidate.values[index]) for candidate in selected} + canonical_values = {_canonical_json(candidate.values[value_index]) for candidate, value_index in selected} if len(canonical_values) != 1: raise AutoConfigCompileError( "ambiguous multi-provider projection {} at slot {}".format( @@ -1230,8 +1346,8 @@ def _compile_config_projections( index, ) ) - selected_values.append(selected[0].values[index]) - selected_provenance.extend(source for candidate in selected for source in candidate.provenance[index]) + selected_values.append(selected[0][0].values[selected[0][1]]) + selected_provenance.extend(source for candidate, value_index in selected for source in candidate.provenance[value_index]) if first.cardinality == ProjectionCardinality.PER_INDEX.value: effective_value = tuple(selected_values) else: diff --git a/apps/predbat/tests/test_lattice_autoconfig.py b/apps/predbat/tests/test_lattice_autoconfig.py index edc4965fc..523f58b6b 100644 --- a/apps/predbat/tests/test_lattice_autoconfig.py +++ b/apps/predbat/tests/test_lattice_autoconfig.py @@ -165,6 +165,40 @@ def projection_value( ) +def partial_projection_snapshot( + provider, + node_id, + index, + argument, + value, + required=True, + identity_aliases=(), +): + """Build one provider-owned aggregate slot and local projection value.""" + roles = tuple( + ProviderRoleAssignment( + role, + "inverters", + index, + node_id, + ) + for role in (AliasRole.PRIMARY, AliasRole.CONTROL) + ) + return projection_snapshot( + provider, + (node_id,), + roles, + ( + config_projection( + argument, + (value,), + required=required, + ), + ), + identity_aliases=identity_aliases, + ) + + def config_projection( argument, values, @@ -524,6 +558,362 @@ def test_reference_only_plan_is_explicitly_shadow_only(self): class TestConfigProjectionCompilation(unittest.TestCase): """Provider contracts project selected indexed capabilities into config.""" + def test_disjoint_providers_compose_partial_indexed_slots(self): + """Each provider can fill its own slot without publishing aggregate width.""" + gateway = partial_projection_snapshot( + "gateway", + "GW1", + 0, + "battery_power", + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway_battery_power", + ), + ) + cloud = partial_projection_snapshot( + "cloud", + "CLOUD1", + 1, + "battery_power", + projection_value( + "CLOUD1", + ProjectionValueKind.ENTITY, + "sensor.cloud_battery_power", + ), + ) + + left = compile_auto_config((gateway, cloud)) + right = compile_auto_config((cloud, gateway)) + argument = left.config_arguments[0] + + self.assertEqual(left.digest, right.digest) + self.assertEqual( + left.projected_config["battery_power"], + ( + "sensor.gateway_battery_power", + "sensor.cloud_battery_power", + ), + ) + self.assertEqual( + { + candidate.provider_id: ( + candidate.slot_indexes, + candidate.target_count, + candidate.values, + ) + for candidate in argument.candidates + }, + { + "gateway": ( + (0,), + 2, + ("sensor.gateway_battery_power",), + ), + "cloud": ( + (1,), + 2, + ("sensor.cloud_battery_power",), + ), + }, + ) + self.assertEqual( + [ + ( + source.field_path, + source.provider_id, + ) + for source in argument.provenance + ], + [ + ("/projected_config/battery_power/0", "gateway"), + ("/projected_config/battery_power/1", "cloud"), + ], + ) + self.assertFalse(left.materialization_readiness.ready) + self.assertEqual( + left.materialization_readiness.blockers, + ("atomic_materializer_missing",), + ) + + def test_provider_owned_slot_disambiguates_repeated_canonical_node(self): + """Explicit roles place correlated providers in their owned slots.""" + identity = "SER-SHARED" + gateway = partial_projection_snapshot( + "gateway", + "GW1", + 0, + "battery_power", + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway_battery_power", + ), + identity_aliases=( + ProviderIdentityAlias( + "serial", + identity, + "GW1", + ), + ), + ) + cloud = partial_projection_snapshot( + "cloud", + "CLOUD1", + 1, + "battery_power", + projection_value( + "CLOUD1", + ProjectionValueKind.ENTITY, + "sensor.cloud_battery_power", + ), + identity_aliases=( + ProviderIdentityAlias( + "serial", + identity, + "CLOUD1", + ), + ), + ) + + plan = compile_auto_config((gateway, cloud)) + + self.assertEqual( + plan.projected_config["battery_power"], + ( + "sensor.gateway_battery_power", + "sensor.cloud_battery_power", + ), + ) + self.assertEqual( + {candidate.provider_id: candidate.slot_indexes for candidate in plan.config_arguments[0].candidates}, + { + "gateway": (0,), + "cloud": (1,), + }, + ) + + def test_repeated_node_projection_ignores_role_declaration_order(self): + """Role tuple order cannot swap values between repeated node slots.""" + roles = tuple( + ProviderRoleAssignment( + role, + "inverters", + index, + "GW1", + ) + for index in (0, 1) + for role in (AliasRole.PRIMARY, AliasRole.CONTROL) + ) + projection = config_projection( + "battery_power", + ( + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway_slot_0", + ), + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway_slot_1", + ), + ), + ) + left = projection_snapshot( + "gateway", + ("GW1",), + roles, + (projection,), + ) + right = projection_snapshot( + "gateway", + ("GW1",), + tuple(reversed(roles)), + (projection,), + ) + + left_plan = compile_auto_config((left,)) + right_plan = compile_auto_config((right,)) + + self.assertEqual(left_plan.digest, right_plan.digest) + self.assertEqual( + left_plan.projected_config["battery_power"], + ( + "sensor.gateway_slot_0", + "sensor.gateway_slot_1", + ), + ) + + def test_unowned_value_cannot_guess_between_repeated_canonical_slots(self): + """A correlated observer must not guess which repeated slot it fills.""" + identity = "SER-AMBIGUOUS" + + def target_provider(provider, node_id, index): + """Build one indexed target without a config projection.""" + roles = tuple( + ProviderRoleAssignment( + role, + "inverters", + index, + node_id, + ) + for role in (AliasRole.PRIMARY, AliasRole.CONTROL) + ) + return projection_snapshot( + provider, + (node_id,), + roles, + (), + identity_aliases=( + ProviderIdentityAlias( + "serial", + identity, + node_id, + ), + ), + ) + + observer = projection_snapshot( + "observer", + ("OBS1",), + (), + ( + config_projection( + "battery_power", + ( + projection_value( + "OBS1", + ProjectionValueKind.ENTITY, + "sensor.observer_battery_power", + ), + ), + ), + ), + identity_aliases=( + ProviderIdentityAlias( + "serial", + identity, + "OBS1", + ), + ), + ) + + with self.assertRaisesRegex( + AutoConfigCompileError, + "ambiguous across aggregate slots", + ): + compile_auto_config( + ( + target_provider("gateway", "GW1", 0), + target_provider("cloud", "CLOUD1", 1), + observer, + ) + ) + + def test_partial_indexed_user_override_covers_aggregate_shape(self): + """A user override resolves the complete array and retains candidates.""" + gateway = partial_projection_snapshot( + "gateway", + "GW1", + 0, + "battery_power", + projection_value( + "GW1", + ProjectionValueKind.ENTITY, + "sensor.gateway_candidate", + ), + ) + cloud = partial_projection_snapshot( + "cloud", + "CLOUD1", + 1, + "battery_power", + projection_value( + "CLOUD1", + ProjectionValueKind.ENTITY, + "sensor.cloud_candidate", + ), + ) + override = UserConfigOverride( + "battery_power", + [ + "sensor.user_slot_0", + "sensor.user_slot_1", + ], + "/apps.yaml/battery_power", + ) + + plan = compile_auto_config( + (gateway, cloud), + user_overrides=(override,), + ) + argument = plan.config_arguments[0] + + self.assertEqual( + plan.projected_config["battery_power"], + ( + "sensor.user_slot_0", + "sensor.user_slot_1", + ), + ) + self.assertEqual( + {source.provider_id for source in argument.candidate_provenance}, + {"gateway", "cloud"}, + ) + self.assertEqual( + [source.provider_id for source in argument.provenance], + ["user_override"], + ) + + def test_optional_partial_projection_preserves_aggregate_hole(self): + """An absent optional slot remains explicit None in the final array.""" + cloud_roles = tuple( + ProviderRoleAssignment( + role, + "inverters", + 1, + "CLOUD1", + ) + for role in (AliasRole.PRIMARY, AliasRole.CONTROL) + ) + gateway = partial_projection_snapshot( + "gateway", + "GW1", + 0, + "pause_mode", + projection_value( + "GW1", + ProjectionValueKind.NONE, + ), + required=False, + ) + cloud = projection_snapshot( + "cloud", + ("CLOUD1",), + cloud_roles, + (), + ) + + plan = compile_auto_config((gateway, cloud)) + + self.assertEqual( + plan.projected_config["pause_mode"], + (None, None), + ) + self.assertEqual( + { + ( + source.field_path, + source.provider_id, + ) + for source in plan.config_arguments[0].candidate_provenance + }, + { + ("/projected_config/pause_mode/0", "gateway"), + }, + ) + def test_gateway_multi_aio_projects_ordered_leaf_arrays(self): """Two Gateway-like AIO nodes produce deterministic per-index arrays.""" nodes = ("AIO1", "AIO2") From 4c3d5d4c670843e93db24bd5c0c6589bc35f2774 Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Tue, 28 Jul 2026 06:48:14 +0100 Subject: [PATCH 5/8] feat(lattice): publish durable compiled config --- apps/predbat/lattice_compiled_publication.py | 756 ++++++++++++ .../test_lattice_compiled_publication.py | 1017 +++++++++++++++++ 2 files changed, 1773 insertions(+) create mode 100644 apps/predbat/lattice_compiled_publication.py create mode 100644 apps/predbat/tests/test_lattice_compiled_publication.py diff --git a/apps/predbat/lattice_compiled_publication.py b/apps/predbat/lattice_compiled_publication.py new file mode 100644 index 000000000..004ff0c27 --- /dev/null +++ b/apps/predbat/lattice_compiled_publication.py @@ -0,0 +1,756 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System - compiled Lattice publication +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +"""Durable immutable publication for compiled Lattice auto-configuration. + +The component in this module is deliberately additive and shadow-only. It +uses the pure compiler's existing single-flight invalidation state machine, +but replaces its future materializer hand-off with one atomic publication. +It has no integration registry, MQTT, Home Assistant, configuration-write, or +other production runtime dependency. +""" + +# cspell:ignore autoconfig + +import hashlib +import threading +from dataclasses import dataclass +from typing import Optional + +from lattice_autoconfig import ( + AutoConfigCompileError, + AutoConfigPlan, + CompileIssue, + CompileRun, + CompileStatus, + LatticeAutoConfigCompiler, + MaterializationReadiness, + UserConfigOverride, + _canonical_json, + _plain, + compile_auto_config, +) +from lattice_topology import TopologyValidationError + + +@dataclass(frozen=True) +class CompilationInvalidation: + """One accepted input cause retained until a successful compile.""" + + source_type: str + source_id: str + generation: int + reason: str + + def __post_init__(self): + """Validate and normalize one auditable invalidation cause.""" + if self.source_type not in ("provider", "user_override"): + raise ValueError("invalidation source_type must be provider or user_override") + if not isinstance(self.source_id, str) or not self.source_id.strip(): + raise ValueError("invalidation source_id must be a non-empty string") + if not isinstance(self.generation, int) or isinstance(self.generation, bool) or self.generation < 0: + raise ValueError("invalidation generation must be a non-negative integer") + if not isinstance(self.reason, str) or not self.reason.strip(): + raise ValueError("invalidation reason must be a non-empty string") + object.__setattr__(self, "source_id", self.source_id.strip()) + object.__setattr__(self, "reason", self.reason.strip()) + + +@dataclass(frozen=True) +class UserOverrideSnapshot: + """One durable generation of explicit runtime user overrides.""" + + generation: int + overrides: tuple + + def __post_init__(self): + """Detach ordering from the reader and reject ambiguous arguments.""" + if not isinstance(self.generation, int) or isinstance(self.generation, bool) or self.generation < 0: + raise ValueError("override generation must be a non-negative integer") + if isinstance(self.overrides, (str, bytes)): + raise ValueError("overrides must be an iterable of UserConfigOverride values") + try: + overrides = tuple(self.overrides) + except TypeError as exc: + raise ValueError("overrides must be an iterable of UserConfigOverride values") from exc + if any(not isinstance(item, UserConfigOverride) for item in overrides): + raise ValueError("overrides must contain UserConfigOverride values") + arguments = [item.argument for item in overrides] + if len(arguments) != len(set(arguments)): + raise ValueError("override arguments must be unique") + object.__setattr__( + self, + "overrides", + tuple(sorted(overrides, key=lambda item: item.argument)), + ) + + +@dataclass(frozen=True) +class CompiledLatticeDiagnostics: + """Immutable status and diagnostics for one compiler observation.""" + + status: CompileStatus + stale: bool + degraded: bool + issues: tuple + readiness: Optional[MaterializationReadiness] + + def __post_init__(self): + """Reject contradictory status flags or mutable diagnostic inputs.""" + if not isinstance(self.status, CompileStatus): + raise ValueError("diagnostic status must be a CompileStatus") + if not isinstance(self.stale, bool) or not isinstance(self.degraded, bool): + raise ValueError("diagnostic flags must be booleans") + if self.stale != (self.status is CompileStatus.STALE): + raise ValueError("stale flag must match STALE status") + if self.degraded != (self.status is CompileStatus.DEGRADED): + raise ValueError("degraded flag must match DEGRADED status") + issues = tuple(self.issues) + if any(not isinstance(issue, CompileIssue) for issue in issues): + raise ValueError("diagnostic issues must contain CompileIssue values") + if self.readiness is not None and not isinstance( + self.readiness, + MaterializationReadiness, + ): + raise ValueError("diagnostic readiness must be MaterializationReadiness or None") + object.__setattr__(self, "issues", issues) + + +@dataclass(frozen=True) +class CompiledLatticePublication: + """One immutable version with its full durable provider input cursor.""" + + lattice_version: int + digest: str + provider_generations: tuple + provider_fingerprints: tuple + provider_requested_generations: tuple + user_override_generation: Optional[int] + user_override_fingerprint: Optional[str] + user_override_requested_generation: Optional[int] + invalidation_causes: tuple + feedback_token: str + diagnostics: CompiledLatticeDiagnostics + plan: AutoConfigPlan + + def __post_init__(self): + """Validate publication integrity and normalize deterministic tuples.""" + if not isinstance(self.lattice_version, int) or isinstance(self.lattice_version, bool) or self.lattice_version < 1: + raise ValueError("lattice_version must be a positive integer") + if not isinstance(self.digest, str) or not self.digest: + raise ValueError("publication digest must be a non-empty string") + if not isinstance(self.plan, AutoConfigPlan): + raise ValueError("publication plan must be an AutoConfigPlan") + if self.plan.digest != self.digest: + raise ValueError("publication digest must match its plan") + + provider_generations = tuple(self.provider_generations) + if any(not isinstance(provider_id, str) or not provider_id or not isinstance(generation, int) or isinstance(generation, bool) or generation < 0 for provider_id, generation in provider_generations): + raise ValueError("publication provider generations are invalid") + if provider_generations != tuple(sorted(set(provider_generations))): + raise ValueError("publication provider generations must be unique and sorted") + durable_generations = dict(provider_generations) + plan_generations = tuple(self.plan.provider_generations) + if any(durable_generations.get(provider_id) != generation for provider_id, generation in plan_generations): + raise ValueError("plan provider generations must be a consistent subset of the durable cursor") + + provider_fingerprints = tuple(self.provider_fingerprints) + fingerprint_keys = [] + for item in provider_fingerprints: + if not isinstance(item, tuple) or len(item) != 3: + raise ValueError("provider fingerprints must contain provider/generation/digest tuples") + provider_id, generation, fingerprint = item + if not isinstance(provider_id, str) or not provider_id or not isinstance(generation, int) or isinstance(generation, bool) or generation < 0 or not isinstance(fingerprint, str) or not fingerprint: + raise ValueError("publication provider fingerprint is invalid") + fingerprint_keys.append((provider_id, generation)) + if tuple(fingerprint_keys) != provider_generations: + raise ValueError("publication provider fingerprints must match provider generations") + + provider_requested_generations = tuple(self.provider_requested_generations) + if any(not isinstance(provider_id, str) or not provider_id or not isinstance(generation, int) or isinstance(generation, bool) or generation < 0 for provider_id, generation in provider_requested_generations): + raise ValueError("publication provider requested generations are invalid") + if provider_requested_generations != tuple(sorted(set(provider_requested_generations))): + raise ValueError("publication provider requested generations must be unique and sorted") + requested_by_provider = dict(provider_requested_generations) + if any(requested_by_provider.get(provider_id, -1) < generation for provider_id, generation in provider_generations): + raise ValueError("provider requested generations must cover the observed cursor") + + if (self.user_override_generation is None) != (self.user_override_fingerprint is None): + raise ValueError("override generation and fingerprint must both be present or absent") + if self.user_override_generation is not None: + if not isinstance(self.user_override_generation, int) or isinstance(self.user_override_generation, bool) or self.user_override_generation < 0 or not isinstance(self.user_override_fingerprint, str) or not self.user_override_fingerprint: + raise ValueError("publication override generation is invalid") + if self.user_override_generation is not None and self.user_override_requested_generation is None: + raise ValueError("observed override cursor requires a requested generation") + if self.user_override_requested_generation is not None: + if not isinstance(self.user_override_requested_generation, int) or isinstance(self.user_override_requested_generation, bool) or self.user_override_requested_generation < 0: + raise ValueError("publication override requested generation is invalid") + if self.user_override_generation is not None and self.user_override_requested_generation < self.user_override_generation: + raise ValueError("override requested generation must cover the observed cursor") + + causes = tuple(sorted(set(self.invalidation_causes), key=_cause_sort_key)) + if any(not isinstance(cause, CompilationInvalidation) for cause in causes): + raise ValueError("publication invalidation causes must contain CompilationInvalidation values") + if not isinstance(self.feedback_token, str) or not self.feedback_token.strip(): + raise ValueError("publication feedback_token must be a non-empty string") + if not isinstance(self.diagnostics, CompiledLatticeDiagnostics): + raise ValueError("publication diagnostics must be CompiledLatticeDiagnostics") + if self.diagnostics.status not in ( + CompileStatus.FRESH, + CompileStatus.DEGRADED, + ): + raise ValueError("only successful compile diagnostics may be published") + if self.diagnostics.stale: + raise ValueError("a successful immutable publication cannot be stale") + if self.diagnostics.readiness != getattr( + self.plan, + "materialization_readiness", + None, + ): + raise ValueError("publication readiness must match its plan") + + object.__setattr__(self, "provider_generations", provider_generations) + object.__setattr__(self, "provider_fingerprints", provider_fingerprints) + object.__setattr__(self, "provider_requested_generations", provider_requested_generations) + object.__setattr__(self, "invalidation_causes", causes) + object.__setattr__(self, "feedback_token", self.feedback_token.strip()) + + +@dataclass(frozen=True) +class CompiledLatticeRun(CompileRun): + """One drain result plus durable publication and live diagnostics.""" + + publication: Optional[CompiledLatticePublication] + published: bool + diagnostics: CompiledLatticeDiagnostics + causes: tuple + + def __post_init__(self): + """Validate the additive publication-specific run fields.""" + if self.publication is not None and not isinstance( + self.publication, + CompiledLatticePublication, + ): + raise ValueError("run publication must be CompiledLatticePublication or None") + if not isinstance(self.published, bool): + raise ValueError("run published flag must be a boolean") + if not isinstance(self.diagnostics, CompiledLatticeDiagnostics): + raise ValueError("run diagnostics must be CompiledLatticeDiagnostics") + causes = tuple(sorted(set(self.causes), key=_cause_sort_key)) + if any(not isinstance(cause, CompilationInvalidation) for cause in causes): + raise ValueError("run causes must contain CompilationInvalidation values") + object.__setattr__(self, "causes", causes) + + +class CompiledLatticeStateStore: + """Injected durable compare-and-swap publication store contract.""" + + def load(self): + """Return the current CompiledLatticePublication or None.""" + raise NotImplementedError + + def compare_and_publish(self, expected_version, publication): + """Atomically publish when the durable version equals expected_version.""" + raise NotImplementedError + + +class InMemoryCompiledLatticeStateStore(CompiledLatticeStateStore): + """Thread-safe reference store for tests; deliberately not durable.""" + + def __init__(self, publication=None): + """Create a store optionally seeded with a validated publication.""" + if publication is not None and not isinstance( + publication, + CompiledLatticePublication, + ): + raise ValueError("initial publication must be CompiledLatticePublication or None") + self._lock = threading.RLock() + self._publication = publication + self._writes = 0 + + @property + def writes(self): + """Return the number of successful atomic publications.""" + with self._lock: + return self._writes + + def load(self): + """Return the immutable current publication.""" + with self._lock: + return self._publication + + def compare_and_publish(self, expected_version, publication): + """Atomically publish exactly the next monotonic version.""" + if not isinstance(expected_version, int) or isinstance(expected_version, bool) or expected_version < 0: + raise ValueError("expected_version must be a non-negative integer") + if not isinstance(publication, CompiledLatticePublication): + raise ValueError("publication must be a CompiledLatticePublication") + if publication.lattice_version != expected_version + 1: + raise ValueError("publication version must be exactly expected_version + 1") + with self._lock: + current_version = self._publication.lattice_version if self._publication is not None else 0 + if current_version != expected_version: + return False + self._publication = publication + self._writes += 1 + return True + + +def _cause_sort_key(cause): + """Return deterministic ordering for generic compiler invalidations.""" + return ( + cause.source_type, + cause.source_id, + cause.generation, + cause.reason, + ) + + +def _fingerprint_overrides(snapshot): + """Bind one override generation to its exact immutable contents.""" + payload = { + "generation": snapshot.generation, + "overrides": [ + { + "argument": item.argument, + "value": _plain(item.value), + "source_path": item.source_path, + } + for item in snapshot.overrides + ], + } + return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() + + +def _same_durable_input(left, right): + """Compare successful durable input cursors and observable diagnostics.""" + return ( + left.digest == right.digest + and left.provider_generations == right.provider_generations + and left.provider_fingerprints == right.provider_fingerprints + and left.provider_requested_generations == right.provider_requested_generations + and left.user_override_generation == right.user_override_generation + and left.user_override_fingerprint == right.user_override_fingerprint + and left.user_override_requested_generation == right.user_override_requested_generation + and left.diagnostics == right.diagnostics + ) + + +def _causes_represented(causes, publication): + """Return whether one publication already audits every local cause.""" + return set(causes).issubset(set(publication.invalidation_causes)) + + +class CompiledLatticeCompiler(LatticeAutoConfigCompiler): + """Dedicated shadow compiler that owns durable Lattice publication.""" + + _OVERRIDE_SOURCE_ID = "user-overrides" + + def __init__( + self, + readers=None, + state_store=None, + override_reader=None, + ): + """Create a publisher over registered fragment and override readers.""" + if state_store is None: + raise ValueError("a durable compiled-Lattice state_store is required") + if not callable(getattr(state_store, "load", None)) or not callable(getattr(state_store, "compare_and_publish", None)): + raise ValueError("state_store must provide load and compare_and_publish") + if override_reader is not None and not callable(override_reader): + raise ValueError("override_reader must be callable") + + super().__init__(readers=readers, materializer=None) + self._state_store = state_store + self._override_reader = override_reader + self._override_requested_generation = -1 + self._override_observed_generation = -1 + self._override_generation_fingerprints = {} + self._pending_causes = set() + self._candidate_issues = () + self._candidate_override_generation = None + self._candidate_override_fingerprint = None + self._publication = None + self._drain_local = threading.local() + + loaded = self._state_store.load() + if loaded is not None and not isinstance( + loaded, + CompiledLatticePublication, + ): + raise ValueError("state_store.load must return CompiledLatticePublication or None") + if loaded is not None: + if (loaded.user_override_generation is not None or loaded.user_override_requested_generation is not None) and self._override_reader is None: + raise ValueError("a published user-override input requires override_reader") + with self._lock: + self._install_publication(loaded) + + @property + def publication(self): + """Return the last durable immutable compiled-Lattice publication.""" + with self._lock: + return self._publication + + @property + def diagnostics(self): + """Return current live diagnostics without mutating the publication.""" + with self._lock: + readiness = self._publication.plan.materialization_readiness if self._publication is not None else None + return CompiledLatticeDiagnostics( + status=self._status, + stale=self._status is CompileStatus.STALE, + degraded=self._status is CompileStatus.DEGRADED, + issues=self._issues, + readiness=readiness, + ) + + def invalidate( + self, + provider_id, + generation, + reason, + feedback_token=None, + ): + """Invalidate any registered provider and retain its audit cause.""" + with self._lock: + accepted = super().invalidate( + provider_id, + generation, + reason, + feedback_token, + ) + if accepted: + self._pending_causes.add( + CompilationInvalidation( + "provider", + provider_id, + generation, + reason, + ) + ) + return accepted + + def invalidate_user_overrides( + self, + generation, + reason, + feedback_token=None, + ): + """Invalidate the durable user-override input monotonically.""" + if self._override_reader is None: + raise RuntimeError("user override reader is not configured") + if not isinstance(generation, int) or isinstance(generation, bool) or generation < 0: + raise ValueError("generation must be a non-negative integer") + if not isinstance(reason, str) or not reason.strip(): + raise ValueError("reason must be a non-empty string") + with self._lock: + if feedback_token is not None and feedback_token in self._feedback_tokens: + return False + previous = max( + self._override_requested_generation, + self._override_observed_generation, + ) + if generation <= previous: + return False + self._override_requested_generation = generation + self._pending_causes.add( + CompilationInvalidation( + "user_override", + self._OVERRIDE_SOURCE_ID, + generation, + reason, + ) + ) + if self._compiling: + self._follow_up = True + else: + self._pending = True + return True + + def _read_user_overrides(self): + """Fresh-read and validate the complete optional override snapshot.""" + if self._override_reader is None: + self._candidate_override_generation = None + self._candidate_override_fingerprint = None + return (), () + try: + snapshot = self._override_reader() + if not isinstance(snapshot, UserOverrideSnapshot): + raise ValueError("override reader must return UserOverrideSnapshot") + fingerprint = _fingerprint_overrides(snapshot) + with self._lock: + required_generation = self._override_requested_generation + previous_generation = self._override_observed_generation + previous_fingerprint = self._override_generation_fingerprints.get(snapshot.generation) + if snapshot.generation < previous_generation: + raise ValueError( + "override generation {} regressed from {}".format( + snapshot.generation, + previous_generation, + ) + ) + if previous_fingerprint is not None and previous_fingerprint != fingerprint: + raise ValueError("override generation {} was reused with different content".format(snapshot.generation)) + if snapshot.generation < required_generation: + raise ValueError( + "override generation {} is behind invalidation {}".format( + snapshot.generation, + required_generation, + ) + ) + self._override_observed_generation = snapshot.generation + self._override_requested_generation = max( + required_generation, + snapshot.generation, + ) + self._override_generation_fingerprints[snapshot.generation] = fingerprint + self._candidate_override_generation = snapshot.generation + self._candidate_override_fingerprint = fingerprint + return snapshot.overrides, () + except (TypeError, ValueError) as exc: + return (), ( + CompileIssue( + "user_overrides_invalid", + str(exc), + self._OVERRIDE_SOURCE_ID, + ), + ) + except Exception as exc: + return (), ( + CompileIssue( + "user_overrides_read_failed", + "{}: {}".format(type(exc).__name__, exc), + self._OVERRIDE_SOURCE_ID, + ), + ) + + def _compile_attempt(self): + """Fresh-read every input and compile exactly one candidate plan.""" + snapshots, provider_issues = self._read_all() + overrides, override_issues = self._read_user_overrides() + issues = provider_issues + override_issues + if override_issues: + detail = "durable user overrides are unavailable or invalid" + self._candidate_issues = issues + (CompileIssue("compile_failed", detail),) + return None, self._candidate_issues + + with self._lock: + active_providers = set(dict(self._active_plan.provider_generations)) if self._active_plan is not None else set() + usable_providers = {snapshot.provider_id for snapshot in snapshots} + unavailable_active = sorted(active_providers - usable_providers) + if unavailable_active: + detail = "previously active provider(s) unavailable: {}".format(", ".join(unavailable_active)) + self._candidate_issues = issues + ( + CompileIssue("active_provider_unavailable", detail), + CompileIssue("compile_failed", detail), + ) + return None, self._candidate_issues + + try: + plan = compile_auto_config(snapshots, overrides) + except ( + AutoConfigCompileError, + TopologyValidationError, + TypeError, + ValueError, + ) as exc: + self._candidate_issues = issues + (CompileIssue("compile_failed", str(exc)),) + return None, self._candidate_issues + self._candidate_issues = issues + return plan, issues + + def _materialize_if_changed(self, plan): + """Publish each changed durable input cursor without materializing.""" + with self._lock: + causes = tuple(sorted(self._pending_causes, key=_cause_sort_key)) + self._drain_local.hook_called = True + self._drain_local.causes = causes + + expected_version = self._publication.lattice_version if self._publication is not None else 0 + lattice_version = expected_version + 1 + feedback_token = "lattice-publication-{}-{}".format( + lattice_version, + plan.digest[:16], + ) + status = CompileStatus.DEGRADED if self._candidate_issues else CompileStatus.FRESH + provider_generations, provider_fingerprints, provider_requested_generations = self._durable_provider_cursor() + diagnostics = CompiledLatticeDiagnostics( + status=status, + stale=False, + degraded=status is CompileStatus.DEGRADED, + issues=self._candidate_issues, + readiness=plan.materialization_readiness, + ) + publication = CompiledLatticePublication( + lattice_version=lattice_version, + digest=plan.digest, + provider_generations=provider_generations, + provider_fingerprints=provider_fingerprints, + provider_requested_generations=provider_requested_generations, + user_override_generation=self._candidate_override_generation, + user_override_fingerprint=self._candidate_override_fingerprint, + user_override_requested_generation=(self._override_requested_generation if self._override_reader is not None and self._override_requested_generation >= 0 else None), + invalidation_causes=causes, + feedback_token=feedback_token, + diagnostics=diagnostics, + plan=plan, + ) + if self._publication is not None and _same_durable_input(self._publication, publication) and _causes_represented(causes, self._publication): + self._active_plan = self._publication.plan + self._pending_causes.clear() + return 0, () + self._feedback_tokens.append(feedback_token) + + try: + committed = self._state_store.compare_and_publish( + expected_version, + publication, + ) + except Exception as exc: + recovered = self._recover_ambiguous_publication(publication) + if recovered: + self._pending_causes.clear() + self._drain_local.published = True + return 0, () + detail = "{}: {}".format(type(exc).__name__, exc) + return 0, ( + CompileIssue("publication_failed", detail), + CompileIssue( + "compile_failed", + "compiled-Lattice publication failed", + ), + ) + + if committed is not True: + try: + current = self._load_current_publication() + except Exception as exc: + detail = "{}: {}".format(type(exc).__name__, exc) + return 0, ( + CompileIssue("publication_failed", detail), + CompileIssue( + "compile_failed", + "compiled-Lattice publication state could not be reloaded", + ), + ) + if current is not None and _same_durable_input(current, publication): + if _causes_represented(causes, current): + self._pending_causes.clear() + else: + self._pending = True + return 0, () + return 0, ( + CompileIssue( + "publication_conflict", + "durable lattice_version changed before atomic publication", + ), + CompileIssue( + "compile_failed", + "compiled-Lattice publication conflicted", + ), + ) + + self._install_publication(publication) + self._pending_causes.clear() + self._drain_local.published = True + return 0, () + + def _recover_ambiguous_publication(self, candidate): + """Recover only when the store contains the exact attempted publication.""" + try: + current = self._load_current_publication() + except Exception: + return False + return current == candidate + + def _durable_provider_cursor(self): + """Return prior-or-fresh fingerprints for current registered readers.""" + provider_generations = [] + provider_fingerprints = [] + provider_requested_generations = [] + for provider_id in sorted(self._readers): + generation = self._observed_generations.get(provider_id) + if generation is not None: + fingerprint = self._generation_fingerprints.get((provider_id, generation)) + if fingerprint is not None: + provider_generations.append((provider_id, generation)) + provider_fingerprints.append((provider_id, generation, fingerprint)) + requested_generation = self._requested_generations.get(provider_id) + if requested_generation is not None and requested_generation >= 0: + provider_requested_generations.append((provider_id, requested_generation)) + return ( + tuple(provider_generations), + tuple(provider_fingerprints), + tuple(provider_requested_generations), + ) + + def _load_current_publication(self): + """Load and install the durable publication after a CAS conflict.""" + current = self._state_store.load() + if current is not None and not isinstance( + current, + CompiledLatticePublication, + ): + raise ValueError("state_store.load must return CompiledLatticePublication or None") + if current is not None: + self._install_publication(current) + return current + + def _install_publication(self, publication): + """Restore one validated publication into compiler safety state.""" + self._publication = publication + self._active_plan = publication.plan + self._status = publication.diagnostics.status + self._issues = publication.diagnostics.issues + for provider_id, generation, fingerprint in publication.provider_fingerprints: + self._observed_generations[provider_id] = generation + self._generation_fingerprints[(provider_id, generation)] = fingerprint + for provider_id, generation in publication.provider_requested_generations: + self._requested_generations[provider_id] = generation + if publication.user_override_generation is not None: + generation = publication.user_override_generation + self._override_observed_generation = generation + self._override_generation_fingerprints[generation] = publication.user_override_fingerprint + if publication.user_override_requested_generation is not None: + self._override_requested_generation = publication.user_override_requested_generation + self._feedback_tokens.append(publication.feedback_token) + + def drain(self): + """Drain once, preserving publication-plan identity in the result.""" + self._drain_local.published = False + self._drain_local.hook_called = False + with self._lock: + self._drain_local.causes = tuple(sorted(self._pending_causes, key=_cause_sort_key)) + run = super().drain() + if not self._drain_local.hook_called: + with self._lock: + self._drain_local.causes = tuple(sorted(self._pending_causes, key=_cause_sort_key)) + with self._lock: + publication = self._publication + if publication is not None: + self._active_plan = publication.plan + active_plan = publication.plan + else: + active_plan = run.plan + diagnostics = CompiledLatticeDiagnostics( + status=run.status, + stale=run.status is CompileStatus.STALE, + degraded=run.status is CompileStatus.DEGRADED, + issues=run.issues, + readiness=(publication.plan.materialization_readiness if publication is not None else None), + ) + return CompiledLatticeRun( + run.attempts, + run.status, + active_plan, + run.issues, + run.invalidations, + run.materializations, + run.pending, + publication, + bool(getattr(self._drain_local, "published", False)), + diagnostics, + tuple(getattr(self._drain_local, "causes", ())), + ) diff --git a/apps/predbat/tests/test_lattice_compiled_publication.py b/apps/predbat/tests/test_lattice_compiled_publication.py new file mode 100644 index 000000000..afdb205ae --- /dev/null +++ b/apps/predbat/tests/test_lattice_compiled_publication.py @@ -0,0 +1,1017 @@ +"""Tests for immutable durable publication of compiled Lattice state.""" + +# cspell:ignore autoconfig + +import os +import sys +import threading +import unittest +from dataclasses import FrozenInstanceError, replace + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from lattice_autoconfig import ( # noqa: E402 + AliasRole, + CompileStatus, + ProjectionCardinality, + ProjectionValueKind, + ProviderHealth, + ProviderSnapshot, + UserConfigOverride, +) +from lattice_compiled_publication import ( # noqa: E402 + CompilationInvalidation, + CompiledLatticeCompiler, + InMemoryCompiledLatticeStateStore, + UserOverrideSnapshot, +) +from tests.test_lattice_autoconfig import ( # noqa: E402 + MutableReader, + config_projection, + fragment, + indexed_roles, + projection_snapshot, + projection_value, + snapshot, +) + + +class MutableOverrideReader: + """Mutable complete override snapshot reader with a call counter.""" + + def __init__(self, value): + """Store the first immutable override generation.""" + self.value = value + self.calls = 0 + + def __call__(self): + """Return the current override generation and count the fresh read.""" + self.calls += 1 + return self.value + + +class RejectOnceStore(InMemoryCompiledLatticeStateStore): + """Reference store that rejects its first atomic publication.""" + + def __init__(self): + """Create one rejecting reference store.""" + super().__init__() + self.reject = True + + def compare_and_publish(self, expected_version, publication): + """Reject once, then delegate to the atomic reference store.""" + if self.reject: + self.reject = False + return False + return super().compare_and_publish(expected_version, publication) + + +class AmbiguousDifferentCursorStore(InMemoryCompiledLatticeStateStore): + """Store that commits a different same-version publication, then raises.""" + + def compare_and_publish(self, expected_version, publication): + """Expose ambiguous-write recovery to a same-digest cursor collision.""" + conflicting = replace( + publication, + provider_generations=publication.provider_generations + (("offline", 7),), + provider_fingerprints=publication.provider_fingerprints + (("offline", 7, "f" * 64),), + provider_requested_generations=publication.provider_requested_generations + (("offline", 7),), + invalidation_causes=publication.invalidation_causes + + ( + CompilationInvalidation( + "provider", + "offline", + 7, + "different committed cursor", + ), + ), + ) + with self._lock: + self._publication = conflicting + self._writes += 1 + raise RuntimeError("connection lost after another cursor committed") + + +class AmbiguousExactStore(InMemoryCompiledLatticeStateStore): + """Store that commits the exact candidate and then loses acknowledgement.""" + + def compare_and_publish(self, expected_version, publication): + """Commit the exact publication before raising an ambiguous error.""" + with self._lock: + self._publication = publication + self._writes += 1 + raise RuntimeError("connection lost after exact commit") + + +def override_projection_snapshot(generation=1): + """Build one provider projection suitable for explicit override tests.""" + return projection_snapshot( + "gateway", + ("GW1",), + indexed_roles(("GW1",)), + ( + config_projection( + "battery_rate_max", + ( + projection_value( + "GW1", + ProjectionValueKind.CONSTANT, + 5, + ), + ), + role=AliasRole.PRIMARY, + cardinality=ProjectionCardinality.PER_INDEX, + ), + ), + generation=generation, + ) + + +def overrides(generation, value): + """Build one complete immutable user override generation.""" + return UserOverrideSnapshot( + generation, + ( + UserConfigOverride( + "battery_rate_max", + [value], + "user.battery_rate_max", + ), + ), + ) + + +class TestCompiledLatticePublication(unittest.TestCase): + """Publication versions each changed durable successful input cursor.""" + + def test_generation_cursor_and_changed_digest_each_publish_once( + self, + ): + """Generation-only and semantic changes each advance lattice_version.""" + reader = MutableReader(snapshot("gateway", generation=1)) + store = InMemoryCompiledLatticeStateStore() + compiler = CompiledLatticeCompiler( + {"gateway": reader}, + state_store=store, + ) + + first = compiler.drain() + + self.assertTrue(first.published) + self.assertEqual(first.publication.lattice_version, 1) + self.assertEqual(store.writes, 1) + + reader.value = snapshot("gateway", generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "generation bookkeeping")) + unchanged = compiler.drain() + + self.assertTrue(unchanged.published) + self.assertEqual(unchanged.publication.lattice_version, 2) + self.assertEqual(unchanged.publication.digest, first.publication.digest) + self.assertEqual(store.writes, 2) + self.assertEqual( + dict(unchanged.plan.provider_generations), + {"gateway": 2}, + ) + self.assertIs(compiler.active_plan, unchanged.publication.plan) + + exact_repeat = compiler.drain() + + self.assertEqual(exact_repeat.attempts, 0) + self.assertFalse(exact_repeat.published) + self.assertIs(exact_repeat.publication, unchanged.publication) + self.assertIs(exact_repeat.plan, unchanged.publication.plan) + self.assertIs(compiler.active_plan, unchanged.publication.plan) + self.assertEqual(store.writes, 2) + + reader.value = snapshot( + "gateway", + generation=3, + node_id="INV2", + ) + self.assertTrue(compiler.invalidate("gateway", 3, "new inverter identity")) + changed = compiler.drain() + + self.assertTrue(changed.published) + self.assertEqual(changed.publication.lattice_version, 3) + self.assertEqual(store.writes, 3) + self.assertNotEqual( + changed.publication.digest, + first.publication.digest, + ) + self.assertEqual( + changed.publication.invalidation_causes, + ( + CompilationInvalidation( + "provider", + "gateway", + 3, + "new inverter identity", + ), + ), + ) + + def test_generation_only_publication_restores_reuse_protection(self): + """Restart restores the newest cursor even when its digest was unchanged.""" + reader = MutableReader(snapshot("gateway", generation=1)) + store = InMemoryCompiledLatticeStateStore() + compiler = CompiledLatticeCompiler( + {"gateway": reader}, + state_store=store, + ) + compiler.drain() + + reader.value = snapshot("gateway", generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "heartbeat cursor")) + second = compiler.drain() + + self.assertTrue(second.published) + self.assertEqual(second.publication.lattice_version, 2) + self.assertEqual( + dict(second.publication.provider_generations), + {"gateway": 2}, + ) + + restarted = CompiledLatticeCompiler( + {"gateway": reader}, + state_store=store, + ) + reader.value = snapshot( + "gateway", + generation=2, + node_id="MUTATED", + ) + failed = restarted.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertIs(failed.publication, second.publication) + self.assertIs(restarted.active_plan, second.publication.plan) + self.assertIn( + "provider_invalid", + {issue.code for issue in failed.issues}, + ) + + def test_offline_provider_cursor_is_durable_but_not_a_plan_contributor(self): + """Offline inputs advance the cursor and retain restart reuse safety.""" + gateway = MutableReader(snapshot("gateway", generation=1)) + cloud = MutableReader( + snapshot( + "cloud", + generation=1, + node_id="CLOUD1", + health=ProviderHealth.OFFLINE, + ) + ) + store = InMemoryCompiledLatticeStateStore() + compiler = CompiledLatticeCompiler( + {"gateway": gateway, "cloud": cloud}, + state_store=store, + ) + first = compiler.drain() + + self.assertEqual( + dict(first.publication.provider_generations), + {"cloud": 1, "gateway": 1}, + ) + self.assertEqual( + dict(first.publication.plan.provider_generations), + {"gateway": 1}, + ) + self.assertLessEqual( + set(first.publication.plan.provider_generations), + set(first.publication.provider_generations), + ) + + cloud.value = snapshot( + "cloud", + generation=2, + node_id="CLOUD1", + health=ProviderHealth.OFFLINE, + ) + self.assertTrue(compiler.invalidate("cloud", 2, "offline heartbeat")) + second = compiler.drain() + + self.assertTrue(second.published) + self.assertEqual(second.publication.lattice_version, 2) + self.assertEqual(second.publication.digest, first.publication.digest) + self.assertEqual( + dict(second.publication.provider_generations), + {"cloud": 2, "gateway": 1}, + ) + durable_fingerprints = second.publication.provider_fingerprints + + exact_repeat = compiler.drain() + self.assertEqual(exact_repeat.attempts, 0) + self.assertFalse(exact_repeat.published) + self.assertEqual(store.writes, 2) + + restarted = CompiledLatticeCompiler( + {"gateway": gateway, "cloud": cloud}, + state_store=store, + ) + cloud.value = snapshot( + "cloud", + generation=2, + node_id="MUTATED", + health=ProviderHealth.OFFLINE, + ) + rejected = restarted.drain() + + self.assertEqual(rejected.status, CompileStatus.DEGRADED) + self.assertTrue(rejected.published) + self.assertEqual(rejected.publication.lattice_version, 3) + self.assertEqual( + rejected.publication.provider_fingerprints, + durable_fingerprints, + ) + self.assertIn( + "provider_invalid", + {issue.code for issue in rejected.issues}, + ) + self.assertIs(restarted.active_plan, rejected.publication.plan) + + def test_reader_failure_preserves_prior_cursor_and_requested_generation(self): + """A reader failure cannot erase its cursor or acknowledged invalidation.""" + gateway = MutableReader(snapshot("gateway", generation=1)) + state = { + "fail": False, + "value": snapshot( + "cloud", + generation=1, + health=ProviderHealth.OFFLINE, + ), + } + + def cloud_reader(): + """Return the cloud snapshot unless the simulated API is down.""" + if state["fail"]: + raise RuntimeError("cloud API unavailable") + return state["value"] + + store = InMemoryCompiledLatticeStateStore() + compiler = CompiledLatticeCompiler( + {"gateway": gateway, "cloud": cloud_reader}, + state_store=store, + ) + first = compiler.drain() + first_cloud_fingerprint = first.publication.provider_fingerprints[0] + + state["fail"] = True + self.assertTrue(compiler.invalidate("cloud", 2, "cloud generation announced")) + degraded = compiler.drain() + + self.assertTrue(degraded.published) + self.assertEqual( + dict(degraded.publication.provider_generations), + {"cloud": 1, "gateway": 1}, + ) + self.assertEqual( + degraded.publication.provider_fingerprints[0], + first_cloud_fingerprint, + ) + self.assertIn( + "provider_read_failed", + {issue.code for issue in degraded.issues}, + ) + self.assertEqual( + dict(degraded.publication.provider_requested_generations), + {"cloud": 2, "gateway": 1}, + ) + + gateway.value = snapshot("gateway", generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "unrelated gateway cursor")) + later = compiler.drain() + + self.assertTrue(later.published) + self.assertEqual(later.publication.lattice_version, 3) + self.assertEqual( + dict(later.publication.provider_requested_generations), + {"cloud": 2, "gateway": 2}, + ) + self.assertNotIn( + CompilationInvalidation( + "provider", + "cloud", + 2, + "cloud generation announced", + ), + later.publication.invalidation_causes, + ) + + state["fail"] = False + restarted = CompiledLatticeCompiler( + {"gateway": gateway, "cloud": cloud_reader}, + state_store=store, + ) + behind = restarted.drain() + + self.assertIn( + "provider_invalid", + {issue.code for issue in behind.issues}, + ) + self.assertIn( + "behind invalidation 2", + " ".join(issue.detail for issue in behind.issues), + ) + + def test_restart_filters_publication_only_unregistered_provider_cursor(self): + """Removed readers are not misreported as freshly observed inputs.""" + gateway = MutableReader(snapshot("gateway", generation=1)) + cloud = MutableReader( + snapshot( + "cloud", + generation=1, + health=ProviderHealth.OFFLINE, + ) + ) + store = InMemoryCompiledLatticeStateStore() + CompiledLatticeCompiler( + {"gateway": gateway, "cloud": cloud}, + state_store=store, + ).drain() + + restarted = CompiledLatticeCompiler( + {"gateway": gateway}, + state_store=store, + ) + filtered = restarted.drain() + + self.assertTrue(filtered.published) + self.assertEqual(filtered.publication.lattice_version, 2) + self.assertEqual( + dict(filtered.publication.provider_generations), + {"gateway": 1}, + ) + self.assertEqual( + filtered.publication.provider_generations, + filtered.publication.plan.provider_generations, + ) + + def test_any_provider_or_override_invalidation_fresh_reads_every_input( + self, + ): + """Every accepted cause reads all fragments and the complete overrides.""" + gateway = MutableReader(snapshot("gateway", generation=1)) + cloud = MutableReader(snapshot("cloud", generation=1)) + override_reader = MutableOverrideReader(UserOverrideSnapshot(1, ())) + compiler = CompiledLatticeCompiler( + {"gateway": gateway, "cloud": cloud}, + state_store=InMemoryCompiledLatticeStateStore(), + override_reader=override_reader, + ) + compiler.drain() + + gateway.value = snapshot("gateway", generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "gateway discovery")) + compiler.drain() + + override_reader.value = UserOverrideSnapshot(2, ()) + self.assertTrue(compiler.invalidate_user_overrides(2, "user config saved")) + override_run = compiler.drain() + + self.assertEqual((gateway.calls, cloud.calls), (3, 3)) + self.assertEqual(override_reader.calls, 3) + self.assertIn( + CompilationInvalidation( + "user_override", + "user-overrides", + 2, + "user config saved", + ), + override_run.causes, + ) + + def test_override_is_public_durable_input_and_reuse_fails_closed(self): + """Override cursors publish even before an effective value changes.""" + provider = MutableReader(override_projection_snapshot()) + override_reader = MutableOverrideReader(overrides(1, 7)) + store = InMemoryCompiledLatticeStateStore() + compiler = CompiledLatticeCompiler( + {"gateway": provider}, + state_store=store, + override_reader=override_reader, + ) + + first = compiler.drain() + self.assertEqual( + first.publication.plan.projected_config["battery_rate_max"], + (7,), + ) + self.assertEqual(first.publication.user_override_generation, 1) + + override_reader.value = overrides(2, 7) + self.assertTrue(compiler.invalidate_user_overrides(2, "override cursor changed")) + second = compiler.drain() + + self.assertTrue(second.published) + self.assertEqual(second.publication.lattice_version, 2) + self.assertEqual(second.publication.digest, first.publication.digest) + self.assertEqual(second.publication.user_override_generation, 2) + self.assertEqual( + second.publication.plan.projected_config["battery_rate_max"], + (7,), + ) + + override_reader.value = overrides(3, 8) + self.assertTrue(compiler.invalidate_user_overrides(3, "override value changed")) + third = compiler.drain() + + self.assertTrue(third.published) + self.assertEqual(third.publication.lattice_version, 3) + self.assertNotEqual(third.publication.digest, second.publication.digest) + self.assertEqual( + third.publication.plan.projected_config["battery_rate_max"], + (8,), + ) + + override_reader.value = overrides(3, 9) + provider.value = override_projection_snapshot(generation=2) + self.assertTrue(compiler.invalidate("gateway", 2, "provider refresh")) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertIs(failed.publication, third.publication) + self.assertIn( + "user_overrides_invalid", + {issue.code for issue in failed.issues}, + ) + + def test_failed_compile_retains_lkg_and_causes_until_recovery(self): + """A failed candidate never publishes and its causes survive retry.""" + reader = MutableReader(snapshot("gateway", generation=1)) + compiler = CompiledLatticeCompiler( + {"gateway": reader}, + state_store=InMemoryCompiledLatticeStateStore(), + ) + first = compiler.drain() + + bad = fragment("gateway", 2) + bad["nodes"].append(dict(bad["nodes"][0])) + reader.value = ProviderSnapshot( + "gateway", + 2, + ProviderHealth.HEALTHY, + bad, + ) + self.assertTrue(compiler.invalidate("gateway", 2, "conflicting rediscovery")) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertFalse(failed.published) + self.assertIs(failed.publication, first.publication) + self.assertEqual(first.publication.lattice_version, 1) + + reader.value = snapshot( + "gateway", + generation=3, + node_id="INV2", + ) + self.assertTrue(compiler.invalidate("gateway", 3, "corrected rediscovery")) + recovered = compiler.drain() + + self.assertTrue(recovered.published) + self.assertEqual(recovered.publication.lattice_version, 2) + self.assertEqual( + set(recovered.publication.invalidation_causes), + { + CompilationInvalidation( + "provider", + "gateway", + 2, + "conflicting rediscovery", + ), + CompilationInvalidation( + "provider", + "gateway", + 3, + "corrected rediscovery", + ), + }, + ) + + def test_second_attempt_supersede_retains_all_causes(self): + """A bounded follow-up superseded again publishes nothing until retry.""" + entered_first = threading.Event() + release_first = threading.Event() + entered_second = threading.Event() + release_second = threading.Event() + state = { + "value": snapshot("gateway", generation=1), + "calls": 0, + } + + def reader(): + """Block the first two post-baseline reads after capturing state.""" + state["calls"] += 1 + value = state["value"] + if state["calls"] == 2: + entered_first.set() + release_first.wait(5) + elif state["calls"] == 3: + entered_second.set() + release_second.wait(5) + return value + + compiler = CompiledLatticeCompiler( + {"gateway": reader}, + state_store=InMemoryCompiledLatticeStateStore(), + ) + baseline = compiler.drain().publication + + state["value"] = snapshot( + "gateway", + generation=2, + node_id="INV2", + ) + self.assertTrue(compiler.invalidate("gateway", 2, "generation two")) + result = {} + worker = threading.Thread(target=lambda: result.setdefault("run", compiler.drain())) + worker.start() + self.assertTrue(entered_first.wait(5)) + state["value"] = snapshot( + "gateway", + generation=3, + node_id="INV3", + ) + self.assertTrue(compiler.invalidate("gateway", 3, "generation three")) + release_first.set() + self.assertTrue(entered_second.wait(5)) + state["value"] = snapshot( + "gateway", + generation=4, + node_id="INV4", + ) + self.assertTrue(compiler.invalidate("gateway", 4, "generation four")) + release_second.set() + worker.join(5) + + superseded = result["run"] + self.assertEqual(superseded.status, CompileStatus.STALE) + self.assertFalse(superseded.published) + self.assertIs(superseded.publication, baseline) + self.assertTrue(superseded.pending) + + recovered = compiler.drain() + self.assertTrue(recovered.published) + self.assertEqual( + {(cause.generation, cause.reason) for cause in recovered.publication.invalidation_causes}, + { + (2, "generation two"), + (3, "generation three"), + (4, "generation four"), + }, + ) + + def test_store_conflict_is_fail_closed_and_retryable(self): + """A rejected CAS keeps the LKG and retries the same next version.""" + store = RejectOnceStore() + compiler = CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=store, + ) + + rejected = compiler.drain() + + self.assertEqual(rejected.status, CompileStatus.STALE) + self.assertIsNone(rejected.publication) + self.assertIn( + "publication_conflict", + {issue.code for issue in rejected.issues}, + ) + self.assertTrue(rejected.pending) + + accepted = compiler.drain() + + self.assertTrue(accepted.published) + self.assertEqual(accepted.publication.lattice_version, 1) + self.assertEqual(store.writes, 1) + + def test_concurrent_same_input_cas_installs_exact_durable_plan(self): + """A CAS loser adopts the winner's publication and exact plan object.""" + store = InMemoryCompiledLatticeStateStore() + first_compiler = CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=store, + ) + second_compiler = CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=store, + ) + + winner = first_compiler.drain() + concurrent = second_compiler.drain() + + self.assertTrue(winner.published) + self.assertFalse(concurrent.published) + self.assertEqual(concurrent.status, CompileStatus.FRESH) + self.assertIs(concurrent.publication, winner.publication) + self.assertIs(concurrent.plan, winner.publication.plan) + self.assertIs(second_compiler.active_plan, winner.publication.plan) + self.assertEqual(store.writes, 1) + + def test_concurrent_same_input_retains_each_compiler_cause_once(self): + """A CAS loser publishes a missing local cause in one follow-up version.""" + store = InMemoryCompiledLatticeStateStore() + seed_reader = MutableReader(snapshot("gateway", generation=1)) + CompiledLatticeCompiler( + {"gateway": seed_reader}, + state_store=store, + ).drain() + + first_reader = MutableReader(snapshot("gateway", generation=2)) + second_reader = MutableReader(snapshot("gateway", generation=2)) + first_compiler = CompiledLatticeCompiler( + {"gateway": first_reader}, + state_store=store, + ) + second_compiler = CompiledLatticeCompiler( + {"gateway": second_reader}, + state_store=store, + ) + self.assertTrue(first_compiler.invalidate("gateway", 2, "compiler A discovery")) + self.assertTrue(second_compiler.invalidate("gateway", 2, "compiler B discovery")) + + winner = first_compiler.drain() + adopted = second_compiler.drain() + + self.assertTrue(winner.published) + self.assertEqual(winner.publication.lattice_version, 2) + self.assertIn( + CompilationInvalidation( + "provider", + "gateway", + 2, + "compiler A discovery", + ), + winner.publication.invalidation_causes, + ) + self.assertFalse(adopted.published) + self.assertTrue(adopted.pending) + self.assertIs(adopted.publication, winner.publication) + + retained = second_compiler.drain() + + self.assertTrue(retained.published) + self.assertFalse(retained.pending) + self.assertEqual(retained.publication.lattice_version, 3) + self.assertEqual(retained.publication.digest, winner.publication.digest) + self.assertEqual( + retained.publication.provider_generations, + winner.publication.provider_generations, + ) + self.assertEqual( + retained.publication.invalidation_causes, + ( + CompilationInvalidation( + "provider", + "gateway", + 2, + "compiler B discovery", + ), + ), + ) + + exact_repeat = second_compiler.drain() + + self.assertEqual(exact_repeat.attempts, 0) + self.assertFalse(exact_repeat.published) + self.assertFalse(exact_repeat.pending) + self.assertIs(exact_repeat.publication, retained.publication) + self.assertEqual(store.writes, 3) + + def test_ambiguous_write_recovery_requires_exact_publication(self): + """Same version, digest, and token cannot alias a different cursor.""" + store = AmbiguousDifferentCursorStore() + compiler = CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=store, + ) + + rejected = compiler.drain() + + self.assertEqual(rejected.status, CompileStatus.STALE) + self.assertFalse(rejected.published) + self.assertTrue(rejected.pending) + self.assertEqual( + dict(rejected.publication.provider_generations), + {"gateway": 1, "offline": 7}, + ) + self.assertIn( + "publication_failed", + {issue.code for issue in rejected.issues}, + ) + self.assertIs(compiler.active_plan, rejected.publication.plan) + + def test_ambiguous_write_recovers_only_the_exact_publication(self): + """Exact durable equality safely resolves a lost acknowledgement.""" + store = AmbiguousExactStore() + compiler = CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=store, + ) + + recovered = compiler.drain() + + self.assertEqual(recovered.status, CompileStatus.FRESH) + self.assertTrue(recovered.published) + self.assertFalse(recovered.pending) + self.assertIs(recovered.publication, store.load()) + self.assertIs(compiler.active_plan, recovered.publication.plan) + + def test_restart_restores_generation_fingerprint_and_feedback_safety(self): + """Durable state rejects replay, content reuse, and write feedback.""" + provider = MutableReader(override_projection_snapshot()) + override_reader = MutableOverrideReader(overrides(1, 7)) + store = InMemoryCompiledLatticeStateStore() + first_compiler = CompiledLatticeCompiler( + {"gateway": provider}, + state_store=store, + override_reader=override_reader, + ) + publication = first_compiler.drain().publication + + restarted = CompiledLatticeCompiler( + {"gateway": provider}, + state_store=store, + override_reader=override_reader, + ) + + self.assertFalse(restarted.invalidate("gateway", 1, "replayed generation")) + self.assertFalse( + restarted.invalidate( + "gateway", + 2, + "publication feedback", + publication.feedback_token, + ) + ) + self.assertFalse( + restarted.invalidate_user_overrides( + 2, + "publication feedback", + publication.feedback_token, + ) + ) + + provider.value = snapshot( + "gateway", + generation=1, + node_id="DIFFERENT", + ) + failed = restarted.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertIs(failed.publication, publication) + self.assertIn( + "provider_invalid", + {issue.code for issue in failed.issues}, + ) + + def test_restart_cannot_silently_drop_durable_override_input(self): + """Observed or requested override state requires its reader on restart.""" + provider = MutableReader(override_projection_snapshot()) + override_reader = MutableOverrideReader(overrides(1, 7)) + store = InMemoryCompiledLatticeStateStore() + compiler = CompiledLatticeCompiler( + {"gateway": provider}, + state_store=store, + override_reader=override_reader, + ) + compiler.drain() + + with self.assertRaisesRegex(ValueError, "requires override_reader"): + CompiledLatticeCompiler( + {"gateway": provider}, + state_store=store, + ) + + plain_store = InMemoryCompiledLatticeStateStore() + plain_publication = ( + CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=plain_store, + ) + .drain() + .publication + ) + requested_only = replace( + plain_publication, + user_override_requested_generation=2, + invalidation_causes=( + CompilationInvalidation( + "user_override", + "user-overrides", + 2, + "override generation announced", + ), + ), + ) + + with self.assertRaisesRegex(ValueError, "requires override_reader"): + CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=InMemoryCompiledLatticeStateStore(requested_only), + ) + + def test_publication_is_deeply_immutable(self): + """The public snapshot and its compiled plan cannot be mutated.""" + compiler = CompiledLatticeCompiler( + {"gateway": MutableReader(snapshot("gateway"))}, + state_store=InMemoryCompiledLatticeStateStore(), + ) + publication = compiler.drain().publication + + with self.assertRaises(FrozenInstanceError): + publication.lattice_version = 9 + with self.assertRaises(TypeError): + publication.plan.topology["scope"] = "site" + + def test_live_stale_diagnostics_do_not_mutate_durable_degraded_snapshot( + self, + ): + """A later failure reports STALE beside the unchanged LKG snapshot.""" + gateway = MutableReader(snapshot("gateway", generation=1)) + offline = MutableReader( + snapshot( + "cloud", + generation=1, + health=ProviderHealth.OFFLINE, + ) + ) + compiler = CompiledLatticeCompiler( + {"gateway": gateway, "cloud": offline}, + state_store=InMemoryCompiledLatticeStateStore(), + ) + first = compiler.drain() + + self.assertEqual(first.status, CompileStatus.DEGRADED) + self.assertTrue(first.publication.diagnostics.degraded) + self.assertFalse(first.publication.diagnostics.stale) + + gateway.value = snapshot( + "gateway", + generation=2, + health=ProviderHealth.OFFLINE, + ) + self.assertTrue(compiler.invalidate("gateway", 2, "gateway unavailable")) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertTrue(failed.diagnostics.stale) + self.assertFalse(failed.diagnostics.degraded) + self.assertIs(failed.publication, first.publication) + self.assertEqual( + failed.publication.diagnostics.status, + CompileStatus.DEGRADED, + ) + + def test_publication_feedback_never_loops_for_any_input(self): + """Published feedback tokens suppress provider and override causes.""" + provider = MutableReader(override_projection_snapshot()) + override_reader = MutableOverrideReader(overrides(1, 7)) + holder = {} + + class FeedbackStore(InMemoryCompiledLatticeStateStore): + """Store that echoes publication feedback synchronously.""" + + def compare_and_publish(self, expected_version, publication): + """Commit, then echo the token through both invalidation APIs.""" + committed = super().compare_and_publish( + expected_version, + publication, + ) + compiler = holder["compiler"] + holder["feedback"] = ( + compiler.invalidate( + "gateway", + 2, + "published config observed", + publication.feedback_token, + ), + compiler.invalidate_user_overrides( + 2, + "published config observed", + publication.feedback_token, + ), + ) + return committed + + store = FeedbackStore() + compiler = CompiledLatticeCompiler( + {"gateway": provider}, + state_store=store, + override_reader=override_reader, + ) + holder["compiler"] = compiler + + run = compiler.drain() + + self.assertTrue(run.published) + self.assertEqual(holder["feedback"], (False, False)) + self.assertFalse(run.pending) + self.assertEqual(provider.calls, 1) + self.assertEqual(override_reader.calls, 1) + + +if __name__ == "__main__": + unittest.main() From b6a2cae5e9741fb709ba6d0c5347bdac0a990992 Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Tue, 28 Jul 2026 07:31:42 +0100 Subject: [PATCH 6/8] feat(lattice): add fragment adapter registry --- apps/predbat/lattice_fragment_adapters.py | 535 ++++++++++++++++++ .../tests/test_lattice_fragment_adapters.py | 520 +++++++++++++++++ 2 files changed, 1055 insertions(+) create mode 100644 apps/predbat/lattice_fragment_adapters.py create mode 100644 apps/predbat/tests/test_lattice_fragment_adapters.py diff --git a/apps/predbat/lattice_fragment_adapters.py b/apps/predbat/lattice_fragment_adapters.py new file mode 100644 index 000000000..e8c4623d1 --- /dev/null +++ b/apps/predbat/lattice_fragment_adapters.py @@ -0,0 +1,535 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System - Lattice fragment adapter registry +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +"""Generic durable integration adapters for compiled Lattice fragments. + +This module is deliberately additive and default-off. Nothing discovers or +registers production integrations unless a caller explicitly enables a +``FragmentAdapterRegistry`` and asks it to create a compiler. + +An integration owns one ``DurableFragmentAdapter`` and its durable state store. +The adapter atomically binds every generation to one semantic fingerprint and +one immutable ``ProviderSnapshot``. The registry only discovers the common +``lattice_fragment_adapter()`` surface; it has no provider or brand allow-list. +""" + +# cspell:ignore autoconfig idempotently unsubscribers + +import hashlib +import threading +from dataclasses import dataclass +from types import MappingProxyType +from typing import Optional, Protocol + +from lattice_autoconfig import ( + ProviderHealth, + ProviderSnapshot, + _fingerprint_snapshot, + _plain, +) +from lattice_compiled_publication import CompiledLatticeCompiler + + +class FragmentAdapterError(RuntimeError): + """Base error for fail-closed fragment adapter operations.""" + + +class FragmentAdapterReadError(FragmentAdapterError): + """A durable fragment could not be read or validated.""" + + +class FragmentAdapterConflict(FragmentAdapterError): + """An atomic fragment publication lost to a different durable state.""" + + +class FragmentAdapterRemoved(FragmentAdapterReadError): + """A registered integration has published a durable removal tombstone.""" + + +def _validate_provider_id(provider_id): + """Normalize one provider-owned stable identifier.""" + if not isinstance(provider_id, str) or not provider_id.strip(): + raise ValueError("provider_id must be a non-empty string") + return provider_id.strip() + + +def _validate_generation(generation): + """Validate one monotonically increasing integration generation.""" + if not isinstance(generation, int) or isinstance(generation, bool) or generation < 0: + raise ValueError("generation must be a non-negative integer") + + +def _validate_reason(reason): + """Normalize one auditable invalidation reason.""" + if not isinstance(reason, str) or not reason.strip(): + raise ValueError("reason must be a non-empty string") + return reason.strip() + + +def _semantic_fingerprint(snapshot, removed=False): + """Return the compiler's generation-bound semantic safety fingerprint.""" + fingerprint = _fingerprint_snapshot(snapshot) + if removed: + return hashlib.sha256("removed:{}".format(fingerprint).encode("utf-8")).hexdigest() + return fingerprint + + +@dataclass(frozen=True) +class FragmentAdapterState: + """One integration-owned durable fragment cursor and immutable value. + + ``semantic_fingerprint`` intentionally uses the compiler's exact + generation-bound fingerprint. The pair therefore detects both generation + regression and reuse of one generation for different safety-relevant + content. + """ + + provider_id: str + generation: int + semantic_fingerprint: str + snapshot: ProviderSnapshot + removed: bool = False + + def __post_init__(self): + """Validate that the durable cursor exactly binds its snapshot.""" + provider_id = _validate_provider_id(self.provider_id) + _validate_generation(self.generation) + if not isinstance(self.snapshot, ProviderSnapshot): + raise ValueError("snapshot must be ProviderSnapshot") + if self.snapshot.provider_id != provider_id: + raise ValueError("snapshot provider_id does not match durable state") + if self.snapshot.generation != self.generation: + raise ValueError("snapshot generation does not match durable state") + if not isinstance(self.removed, bool): + raise ValueError("removed must be a boolean") + expected = _semantic_fingerprint(self.snapshot, self.removed) + if self.semantic_fingerprint != expected: + raise ValueError("semantic_fingerprint does not match the immutable snapshot") + object.__setattr__(self, "provider_id", provider_id) + + +class FragmentAdapterStateStore: + """Required atomic durable-store protocol owned by one integration.""" + + def load(self): + """Return the current ``FragmentAdapterState`` or ``None``.""" + raise NotImplementedError + + def compare_and_store(self, expected, replacement): + """Atomically store replacement only when current state equals expected.""" + raise NotImplementedError + + +class FragmentPublisher(Protocol): + """Structural integration protocol discovered without provider knowledge.""" + + provider_id: str + + def read_state(self) -> FragmentAdapterState: + """Fresh-read the integration-owned durable state.""" + ... + + def read_snapshot(self) -> ProviderSnapshot: + """Fresh-read the current immutable provider snapshot.""" + ... + + def subscribe_invalidation(self, listener): + """Attach the registry's invalidation sink and return an unsubscribe.""" + ... + + +class FragmentPublishingComponent(Protocol): + """Structural component discovery surface used by the generic registry.""" + + def lattice_fragment_adapter(self) -> Optional[FragmentPublisher]: + """Return this component's fragment publisher, if it has one.""" + ... + + +class InMemoryFragmentAdapterStateStore(FragmentAdapterStateStore): + """Thread-safe reference state store for tests; not production durability.""" + + def __init__(self, state=None): + """Create a store optionally seeded with one validated state.""" + if state is not None and not isinstance(state, FragmentAdapterState): + raise ValueError("initial state must be FragmentAdapterState or None") + self._lock = threading.RLock() + self._state = state + self._writes = 0 + + @property + def writes(self): + """Return the number of successful atomic writes.""" + with self._lock: + return self._writes + + def load(self): + """Return the immutable current state.""" + with self._lock: + return self._state + + def compare_and_store(self, expected, replacement): + """Atomically compare the complete cursor and install replacement.""" + if replacement is not None and not isinstance( + replacement, + FragmentAdapterState, + ): + raise ValueError("replacement must be FragmentAdapterState or None") + with self._lock: + if self._state != expected: + return False + self._state = replacement + self._writes += 1 + return True + + +class DurableFragmentAdapter: + """Generic publisher over one integration-owned atomic state store.""" + + def __init__(self, provider_id, state_store): + """Restore one provider cursor without performing discovery or writes.""" + self.provider_id = _validate_provider_id(provider_id) + if not callable(getattr(state_store, "load", None)) or not callable(getattr(state_store, "compare_and_store", None)): + raise ValueError("state_store must provide load and compare_and_store") + self._state_store = state_store + self._lock = threading.RLock() + self._listeners = [] + self._validate_loaded_state(self._load()) + + def _load(self): + """Load durable state and convert store faults into adapter faults.""" + try: + return self._state_store.load() + except Exception as exc: + raise FragmentAdapterReadError( + "durable fragment load failed: {}: {}".format( + type(exc).__name__, + exc, + ) + ) from exc + + def _validate_loaded_state(self, state): + """Validate one store result and its provider ownership.""" + if state is None: + return None + if not isinstance(state, FragmentAdapterState): + raise FragmentAdapterReadError("state_store.load must return FragmentAdapterState or None") + try: + state.__post_init__() + except ValueError as exc: + raise FragmentAdapterReadError(str(exc)) from exc + if state.provider_id != self.provider_id: + raise FragmentAdapterReadError("durable state belongs to provider {}".format(state.provider_id)) + return state + + def read_state(self): + """Fresh-read the complete immutable durable fragment state.""" + with self._lock: + state = self._validate_loaded_state(self._load()) + if state is None: + raise FragmentAdapterReadError("provider {} has no durable fragment".format(self.provider_id)) + return state + + def read_snapshot(self): + """Fresh-read one immutable snapshot or fail closed on removal.""" + state = self.read_state() + if state.removed: + raise FragmentAdapterRemoved( + "provider {} was removed at generation {}".format( + self.provider_id, + state.generation, + ) + ) + return state.snapshot + + def subscribe_invalidation(self, listener): + """Subscribe the compiler-facing invalidation sink.""" + if not callable(listener): + raise ValueError("invalidation listener must be callable") + with self._lock: + if listener in self._listeners: + raise ValueError("invalidation listener is already subscribed") + self._listeners.append(listener) + closed = [False] + + def unsubscribe(): + """Detach this exact listener idempotently.""" + with self._lock: + if closed[0]: + return + closed[0] = True + if listener in self._listeners: + self._listeners.remove(listener) + + return unsubscribe + + def publish( + self, + snapshot, + reason, + feedback_token=None, + removed=False, + ): + """Atomically publish a newer fragment and notify every subscriber. + + Invalidation is announced before the durable CAS. Consequently a + failed or conflicting store write leaves the compiler pending and its + fresh reader behind the requested generation, which fails closed. + A subscriber may return ``False`` only to suppress publication-origin + feedback; that token then causes neither persistence nor recompilation. + """ + if not isinstance(snapshot, ProviderSnapshot): + raise ValueError("snapshot must be ProviderSnapshot") + if snapshot.provider_id != self.provider_id: + raise ValueError("snapshot provider_id does not match adapter") + if not isinstance(removed, bool): + raise ValueError("removed must be a boolean") + reason = _validate_reason(reason) + candidate = FragmentAdapterState( + self.provider_id, + snapshot.generation, + _semantic_fingerprint(snapshot, removed), + snapshot, + removed, + ) + + with self._lock: + current = self._validate_loaded_state(self._load()) + if current is not None: + if candidate.generation < current.generation: + raise ValueError( + "fragment generation {} regressed from {}".format( + candidate.generation, + current.generation, + ) + ) + if candidate.generation == current.generation: + if candidate.semantic_fingerprint != (current.semantic_fingerprint): + raise ValueError("fragment generation {} was reused with different " "content".format(candidate.generation)) + return False + listeners = tuple(self._listeners) + + for listener in listeners: + accepted = listener( + self.provider_id, + candidate.generation, + reason, + feedback_token, + ) + if accepted is False: + return False + + try: + committed = self._state_store.compare_and_store( + current, + candidate, + ) + except Exception as exc: + raise FragmentAdapterConflict( + "durable fragment publication failed: {}: {}".format( + type(exc).__name__, + exc, + ) + ) from exc + if committed is not True: + winner = self._validate_loaded_state(self._load()) + if winner == candidate: + return False + raise FragmentAdapterConflict("durable fragment cursor changed before atomic publication") + return True + + def remove(self, generation, reason, feedback_token=None): + """Publish a durable removal tombstone and invalidate the compiler.""" + _validate_generation(generation) + current = self.read_state() + snapshot = current.snapshot + tombstone = ProviderSnapshot( + self.provider_id, + generation, + ProviderHealth.OFFLINE, + _plain(snapshot.topology_fragment), + snapshot.aliases, + snapshot.identity_aliases, + snapshot.role_assignments, + snapshot.config_projections, + ) + return self.publish( + tombstone, + reason, + feedback_token=feedback_token, + removed=True, + ) + + +class FragmentAdapterRegistry: + """Default-off brand-neutral discovery and frozen compiler registry.""" + + DISCOVERY_METHOD = "lattice_fragment_adapter" + + def __init__(self, enabled=False): + """Create an empty registry; disabled is the safe default.""" + if not isinstance(enabled, bool): + raise ValueError("enabled must be a boolean") + self._enabled = enabled + self._lock = threading.RLock() + self._adapters = {} + self._compiler = None + self._unsubscribers = () + self._sealed = False + + @property + def enabled(self): + """Return whether explicit fragment discovery is enabled.""" + return self._enabled + + @property + def provider_ids(self): + """Return registered provider identities in deterministic order.""" + with self._lock: + return tuple(sorted(self._adapters)) + + @property + def readers(self): + """Return an immutable provider-reader mapping for inspection/tests.""" + with self._lock: + return MappingProxyType({provider_id: adapter.read_snapshot for provider_id, adapter in self._adapters.items()}) + + def _validate_adapter(self, adapter): + """Validate the common adapter surface and its durable current state.""" + provider_id = _validate_provider_id(getattr(adapter, "provider_id", None)) + for method_name in ( + "read_state", + "read_snapshot", + "subscribe_invalidation", + ): + if not callable(getattr(adapter, method_name, None)): + raise ValueError("fragment adapter must provide {}".format(method_name)) + state = adapter.read_state() + if not isinstance(state, FragmentAdapterState): + raise ValueError("adapter read_state must return FragmentAdapterState") + if state.provider_id != provider_id: + raise ValueError("adapter state belongs to provider {}".format(state.provider_id)) + return provider_id + + def discover(self, components): + """Discover every component implementing the common publisher surface.""" + if not self._enabled: + return () + candidates = [] + for component in tuple(components): + factory = getattr(component, self.DISCOVERY_METHOD, None) + if factory is None: + continue + if not callable(factory): + raise ValueError("{} must be callable".format(self.DISCOVERY_METHOD)) + adapter = factory() + if adapter is not None: + candidates.append(adapter) + + validated = [] + seen = set() + for adapter in candidates: + provider_id = self._validate_adapter(adapter) + if provider_id in seen: + raise ValueError("provider {} was discovered more than once".format(provider_id)) + seen.add(provider_id) + validated.append((provider_id, adapter)) + + with self._lock: + if self._sealed: + raise RuntimeError("fragment registry membership is sealed") + duplicate = sorted(provider_id for provider_id, _adapter in validated if provider_id in self._adapters) + if duplicate: + raise ValueError("provider {} is already registered".format(duplicate[0])) + for provider_id, adapter in validated: + self._adapters[provider_id] = adapter + return tuple(provider_id for provider_id, _adapter in validated) + + def register(self, adapter): + """Register one explicitly supplied generic adapter before sealing.""" + if not self._enabled: + return False + provider_id = self._validate_adapter(adapter) + with self._lock: + if self._sealed: + raise RuntimeError("fragment registry membership is sealed") + if provider_id in self._adapters: + raise ValueError("provider {} is already registered".format(provider_id)) + self._adapters[provider_id] = adapter + return True + + def unregister(self, provider_id): + """Remove only pre-bind registration; runtime removal is a tombstone.""" + if not self._enabled: + return False + provider_id = _validate_provider_id(provider_id) + with self._lock: + if self._sealed: + raise RuntimeError("runtime unregister is unsafe; publish a durable removal " "tombstone") + if provider_id not in self._adapters: + raise KeyError("unknown provider {}".format(provider_id)) + del self._adapters[provider_id] + return True + + def create_compiler(self, state_store, override_reader=None): + """Freeze membership and create the sole compiled-Lattice coordinator.""" + if not self._enabled: + raise RuntimeError("fragment adapter registry is disabled") + with self._lock: + if self._sealed: + raise RuntimeError("fragment registry membership is already sealed") + if not self._adapters: + raise RuntimeError("cannot create a compiler without fragment adapters") + for adapter in self._adapters.values(): + self._validate_adapter(adapter) + readers = {provider_id: adapter.read_snapshot for provider_id, adapter in self._adapters.items()} + compiler = CompiledLatticeCompiler( + readers, + state_store=state_store, + override_reader=override_reader, + ) + + unsubscribers = [] + try: + for provider_id, adapter in sorted(self._adapters.items()): + + def invalidate( + source_id, + generation, + reason, + feedback_token, + expected_id=provider_id, + ): + """Forward this adapter's invalidation to the compiler.""" + if source_id != expected_id: + raise FragmentAdapterError( + "adapter {} emitted invalidation for {}".format( + expected_id, + source_id, + ) + ) + accepted = compiler.invalidate( + source_id, + generation, + reason, + feedback_token, + ) + if feedback_token is not None and accepted is False: + return False + return True + + unsubscribe = adapter.subscribe_invalidation(invalidate) + if not callable(unsubscribe): + raise ValueError("subscribe_invalidation must return an " "unsubscribe callable") + unsubscribers.append(unsubscribe) + except Exception: + for unsubscribe in reversed(unsubscribers): + unsubscribe() + raise + + self._compiler = compiler + self._unsubscribers = tuple(unsubscribers) + self._sealed = True + return compiler diff --git a/apps/predbat/tests/test_lattice_fragment_adapters.py b/apps/predbat/tests/test_lattice_fragment_adapters.py new file mode 100644 index 000000000..b0de1b4f0 --- /dev/null +++ b/apps/predbat/tests/test_lattice_fragment_adapters.py @@ -0,0 +1,520 @@ +"""Tests for generic durable Lattice fragment adapter discovery.""" + +# cspell:ignore autoconfig + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from lattice_autoconfig import ( # noqa: E402 + CompileStatus, + ProviderHealth, +) +from lattice_compiled_publication import ( # noqa: E402 + InMemoryCompiledLatticeStateStore, +) +from lattice_fragment_adapters import ( # noqa: E402 + DurableFragmentAdapter, + FragmentAdapterConflict, + FragmentAdapterReadError, + FragmentAdapterRegistry, + FragmentAdapterRemoved, + FragmentAdapterState, + InMemoryFragmentAdapterStateStore, +) +from tests.test_lattice_autoconfig import snapshot # noqa: E402 + + +class FragmentComponent: + """Brand-neutral component exposing only the common discovery method.""" + + def __init__(self, adapter): + """Store the adapter returned during discovery.""" + self.adapter = adapter + self.calls = 0 + + def lattice_fragment_adapter(self): + """Return the component-owned fragment publisher.""" + self.calls += 1 + return self.adapter + + +class UnrelatedComponent: + """Component without any Lattice fragment publisher surface.""" + + +class RaisingLoadStore(InMemoryFragmentAdapterStateStore): + """State store whose durable read is unavailable.""" + + def load(self): + """Raise one representative durable-store fault.""" + raise OSError("disk unavailable") + + +class RejectingStore(InMemoryFragmentAdapterStateStore): + """State store rejecting every atomic fragment write.""" + + def compare_and_store(self, expected, replacement): + """Reject the candidate without changing durable state.""" + return False + + +class ToggleLoadStore(InMemoryFragmentAdapterStateStore): + """State store that can fail after an initial successful compilation.""" + + def __init__(self): + """Create an initially available durable store.""" + super().__init__() + self.fail_reads = False + + def load(self): + """Return state until the test makes durable reads unavailable.""" + if self.fail_reads: + raise OSError("durable fragment unavailable") + return super().load() + + +def publisher(provider_id, generation=1, health=ProviderHealth.HEALTHY): + """Build one seeded durable generic publisher.""" + state_store = InMemoryFragmentAdapterStateStore() + adapter = DurableFragmentAdapter(provider_id, state_store) + initial = snapshot( + provider_id, + generation=generation, + node_id="{}-INV".format(provider_id.upper()), + health=health, + ) + adapter.publish(initial, "initial discovery") + return adapter, state_store, initial + + +def advance( + current, + generation, + health=None, + node_id=None, +): + """Build the next immutable test snapshot without copying frozen mappings.""" + if health is None: + health = current.health + if node_id is None: + node_id = current.topology_fragment["nodes"][0]["id"] + return snapshot( + current.provider_id, + generation=generation, + node_id=node_id, + health=health, + ) + + +def compiled_registry(*adapters): + """Discover generic components and create a durable compiler.""" + registry = FragmentAdapterRegistry(enabled=True) + components = [ + UnrelatedComponent(), + ] + [FragmentComponent(adapter) for adapter in adapters] + discovered = registry.discover(components) + compiled_store = InMemoryCompiledLatticeStateStore() + compiler = registry.create_compiler(compiled_store) + return registry, compiler, compiled_store, discovered + + +class TestDurableFragmentAdapter(unittest.TestCase): + """Each integration owns a monotonic durable immutable fragment cursor.""" + + def test_seed_and_fresh_reads_are_immutable_and_durable(self): + """A published snapshot is detached and restored from its store.""" + adapter, state_store, initial = publisher("gateway") + + state = adapter.read_state() + + self.assertIsInstance(state, FragmentAdapterState) + self.assertEqual(state.generation, 1) + self.assertEqual(len(state.semantic_fingerprint), 64) + self.assertIs(state.snapshot, initial) + self.assertIs(adapter.read_snapshot(), initial) + self.assertEqual(state_store.writes, 1) + + restarted = DurableFragmentAdapter("gateway", state_store) + self.assertEqual(restarted.read_state(), state) + self.assertIs(restarted.read_snapshot(), initial) + + def test_restart_rejects_regression_and_generation_reuse(self): + """Durable cursor restoration rejects regressions and mutations.""" + _adapter, state_store, initial = publisher("cloud", generation=7) + restarted = DurableFragmentAdapter("cloud", state_store) + + with self.assertRaisesRegex(ValueError, "regressed"): + restarted.publish( + advance(initial, 6), + "stale cache", + ) + with self.assertRaisesRegex(ValueError, "reused"): + restarted.publish( + advance(initial, 7, node_id="MUTATED"), + "same generation mutation", + ) + self.assertFalse( + restarted.publish(initial, "exact replay"), + ) + self.assertEqual(state_store.writes, 1) + + def test_reader_store_failure_is_wrapped_and_fails_closed(self): + """No synthetic or empty fragment is returned on durable read failure.""" + store = RaisingLoadStore() + + with self.assertRaisesRegex( + FragmentAdapterReadError, + "disk unavailable", + ): + DurableFragmentAdapter("gateway", store) + + def test_conflicting_atomic_write_leaves_requested_generation_pending(self): + """A rejected CAS never presents an uncommitted fragment as current.""" + adapter, seeded_store, initial = publisher("gateway") + rejecting = RejectingStore(adapter.read_state()) + adapter = DurableFragmentAdapter("gateway", rejecting) + _registry, compiler, _compiled_store, _discovered = compiled_registry(adapter) + first = compiler.drain() + self.assertEqual(first.status, CompileStatus.FRESH) + + updated = advance(initial, 2) + with self.assertRaises(FragmentAdapterConflict): + adapter.publish(updated, "new telemetry") + + failed = compiler.drain() + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertTrue(failed.pending) + self.assertEqual(adapter.read_state(), seeded_store.load()) + self.assertEqual( + first.publication, + compiler.publication, + ) + + def test_removal_is_durable_and_reader_fails_closed_after_restart(self): + """Runtime removal is an auditable tombstone, never silent absence.""" + adapter, state_store, _initial = publisher("gateway") + + self.assertTrue(adapter.remove(2, "integration removed")) + removed = adapter.read_state() + + self.assertTrue(removed.removed) + self.assertEqual(removed.generation, 2) + self.assertEqual(removed.snapshot.health, ProviderHealth.OFFLINE) + with self.assertRaises(FragmentAdapterRemoved): + adapter.read_snapshot() + + restarted = DurableFragmentAdapter("gateway", state_store) + self.assertEqual(restarted.read_state(), removed) + with self.assertRaises(FragmentAdapterRemoved): + restarted.read_snapshot() + + +class TestFragmentAdapterRegistry(unittest.TestCase): + """Any common-surface component can drive the compiled coordinator.""" + + def test_registry_is_default_off_and_does_not_touch_components(self): + """Disabled discovery performs no integration calls or registration.""" + adapter, _store, _initial = publisher("gateway") + component = FragmentComponent(adapter) + registry = FragmentAdapterRegistry() + + self.assertEqual(registry.discover([component]), ()) + self.assertEqual(component.calls, 0) + self.assertEqual(registry.provider_ids, ()) + self.assertFalse(registry.register(adapter)) + self.assertFalse(registry.unregister("gateway")) + with self.assertRaisesRegex(RuntimeError, "disabled"): + registry.create_compiler(InMemoryCompiledLatticeStateStore()) + + def test_discovery_has_no_brand_allow_list(self): + """Arbitrary provider IDs are discovered through the common surface.""" + alpha, _alpha_store, _alpha = publisher("future-cloud-alpha") + beta, _beta_store, _beta = publisher("local-modbus-beta") + + registry, compiler, _store, discovered = compiled_registry( + alpha, + beta, + ) + run = compiler.drain() + + self.assertEqual( + discovered, + ("future-cloud-alpha", "local-modbus-beta"), + ) + self.assertEqual( + registry.provider_ids, + ("future-cloud-alpha", "local-modbus-beta"), + ) + self.assertEqual( + dict(run.publication.provider_generations), + { + "future-cloud-alpha": 1, + "local-modbus-beta": 1, + }, + ) + + def test_invalid_discovery_batch_is_transactional(self): + """Duplicate discovery does not partially mutate registry membership.""" + alpha, _alpha_store, _initial = publisher("same-provider") + duplicate = DurableFragmentAdapter( + "same-provider", + _alpha_store, + ) + registry = FragmentAdapterRegistry(enabled=True) + + with self.assertRaisesRegex(ValueError, "more than once"): + registry.discover( + [ + FragmentComponent(alpha), + FragmentComponent(duplicate), + ] + ) + + self.assertEqual(registry.provider_ids, ()) + + def test_register_unregister_only_before_compiler_is_sealed(self): + """Runtime membership changes cannot silently alter compiler inputs.""" + adapter, _store, _initial = publisher("gateway") + registry = FragmentAdapterRegistry(enabled=True) + + self.assertTrue(registry.register(adapter)) + self.assertTrue(registry.unregister("gateway")) + self.assertTrue(registry.register(adapter)) + compiler = registry.create_compiler(InMemoryCompiledLatticeStateStore()) + + with self.assertRaisesRegex(RuntimeError, "tombstone"): + registry.unregister("gateway") + with self.assertRaisesRegex(RuntimeError, "sealed"): + registry.register(adapter) + self.assertEqual(compiler.drain().status, CompileStatus.FRESH) + + def test_any_registered_integration_invalidates_and_republishes(self): + """Every discovered publisher can replace its prior fragment.""" + alpha, _alpha_store, alpha_one = publisher("alpha") + beta, _beta_store, beta_one = publisher("beta") + _registry, compiler, compiled_store, _ids = compiled_registry( + alpha, + beta, + ) + first = compiler.drain() + + alpha_two = advance(alpha_one, 2) + beta_two = advance(beta_one, 2) + self.assertTrue(alpha.publish(alpha_two, "alpha refresh")) + self.assertTrue(beta.publish(beta_two, "beta refresh")) + second = compiler.drain() + + self.assertEqual(second.attempts, 1) + self.assertTrue(second.published) + self.assertEqual(second.publication.lattice_version, 2) + self.assertEqual( + dict(second.publication.provider_generations), + {"alpha": 2, "beta": 2}, + ) + self.assertEqual(compiled_store.writes, 2) + self.assertEqual( + {(cause.source_id, cause.generation) for cause in second.publication.invalidation_causes}, + {("alpha", 2), ("beta", 2)}, + ) + self.assertEqual(first.publication.lattice_version, 1) + + def test_degraded_fragment_invalidates_and_publishes_degraded_cursor(self): + """Health-only changes are durable inputs and trigger recompilation.""" + adapter, _store, initial = publisher("gateway") + _registry, compiler, _compiled_store, _ids = compiled_registry(adapter) + compiler.drain() + + degraded = advance( + initial, + 2, + health=ProviderHealth.DEGRADED, + ) + self.assertTrue(adapter.publish(degraded, "provider health degraded")) + run = compiler.drain() + + self.assertTrue(run.published) + self.assertEqual(run.status, CompileStatus.DEGRADED) + self.assertEqual( + dict(run.publication.provider_generations), + {"gateway": 2}, + ) + self.assertIn( + "provider_degraded", + {issue.code for issue in run.issues}, + ) + + def test_offline_fragment_invalidates_but_preserves_last_known_good(self): + """An active provider going offline fails closed after one attempt.""" + adapter, _store, initial = publisher("gateway") + _registry, compiler, _compiled_store, _ids = compiled_registry(adapter) + first = compiler.drain() + + offline = advance( + initial, + 2, + health=ProviderHealth.OFFLINE, + ) + self.assertTrue(adapter.publish(offline, "provider disconnected")) + run = compiler.drain() + + self.assertEqual(run.attempts, 1) + self.assertEqual(run.status, CompileStatus.STALE) + self.assertTrue(run.pending) + self.assertIs(run.publication, first.publication) + self.assertIs(run.plan, first.publication.plan) + self.assertIn( + "active_provider_unavailable", + {issue.code for issue in run.issues}, + ) + + def test_other_provider_invalidation_exposes_reader_failure_fail_closed(self): + """A fresh-read failure blocks publication instead of dropping input.""" + gateway_store = ToggleLoadStore() + gateway = DurableFragmentAdapter("gateway", gateway_store) + gateway_one = snapshot("gateway", generation=1, node_id="GW-INV") + gateway.publish(gateway_one, "initial gateway discovery") + cloud, _cloud_store, cloud_one = publisher("cloud") + _registry, compiler, _compiled_store, _ids = compiled_registry( + gateway, + cloud, + ) + first = compiler.drain() + + gateway_store.fail_reads = True + self.assertTrue( + cloud.publish( + advance(cloud_one, 2), + "cloud refresh requires fresh read of every provider", + ) + ) + run = compiler.drain() + + self.assertEqual(run.attempts, 1) + self.assertEqual(run.status, CompileStatus.STALE) + self.assertTrue(run.pending) + self.assertIs(run.publication, first.publication) + self.assertIn( + "provider_read_failed", + {issue.code for issue in run.issues}, + ) + + def test_removal_invalidates_and_preserves_last_known_good(self): + """A durable removal triggers a bounded fail-closed recompile.""" + adapter, _store, _initial = publisher("gateway") + _registry, compiler, _compiled_store, _ids = compiled_registry(adapter) + first = compiler.drain() + + self.assertTrue(adapter.remove(2, "integration disabled by user")) + run = compiler.drain() + + self.assertEqual(run.attempts, 1) + self.assertEqual(run.status, CompileStatus.STALE) + self.assertTrue(run.pending) + self.assertIs(run.publication, first.publication) + self.assertIn( + "provider_read_failed", + {issue.code for issue in run.issues}, + ) + + def test_restart_restores_adapter_and_compiler_cursor_protection(self): + """Both durable layers reject reuse after a complete process restart.""" + adapter, adapter_store, initial = publisher("gateway") + registry, compiler, compiled_store, _ids = compiled_registry(adapter) + compiler.drain() + second_snapshot = advance(initial, 2) + self.assertTrue(adapter.publish(second_snapshot, "new discovery")) + second = compiler.drain() + self.assertEqual(second.publication.lattice_version, 2) + + restarted_adapter = DurableFragmentAdapter( + "gateway", + adapter_store, + ) + restarted_registry, restarted_compiler, _same_store, _ids = compiled_registry_with_store( + compiled_store, + restarted_adapter, + ) + exact = restarted_compiler.drain() + + self.assertEqual( + restarted_registry.provider_ids, + registry.provider_ids, + ) + self.assertFalse(exact.published) + self.assertEqual(exact.publication, second.publication) + with self.assertRaisesRegex(ValueError, "reused"): + restarted_adapter.publish( + advance(second_snapshot, 2, node_id="REUSED"), + "same cursor mutation after restart", + ) + self.assertEqual( + restarted_compiler.publication, + second.publication, + ) + + def test_publication_feedback_token_does_not_persist_or_recompile(self): + """Compiler-origin feedback is suppressed before adapter persistence.""" + adapter, state_store, initial = publisher("gateway") + _registry, compiler, compiled_store, _ids = compiled_registry(adapter) + first = compiler.drain() + + feedback_snapshot = advance(initial, 2) + self.assertFalse( + adapter.publish( + feedback_snapshot, + "published config observed", + feedback_token=first.publication.feedback_token, + ) + ) + idle = compiler.drain() + + self.assertEqual(adapter.read_state().generation, 1) + self.assertEqual(state_store.writes, 1) + self.assertEqual(compiled_store.writes, 1) + self.assertEqual(idle.attempts, 0) + self.assertIs(idle.publication, first.publication) + + def test_invalidation_during_read_gets_one_bounded_follow_up(self): + """Concurrent adapter invalidation triggers exactly one fresh follow-up.""" + adapter, _store, initial = publisher("gateway") + registry = FragmentAdapterRegistry(enabled=True) + registry.register(adapter) + original_reader = adapter.read_snapshot + fired = [False] + + def invalidating_reader(): + """Publish one newer generation during the first compile read.""" + value = original_reader() + if not fired[0]: + fired[0] = True + adapter.publish( + advance(initial, 2), + "concurrent rediscovery", + ) + return value + + adapter.read_snapshot = invalidating_reader + compiler = registry.create_compiler(InMemoryCompiledLatticeStateStore()) + run = compiler.drain() + + self.assertEqual(run.attempts, 2) + self.assertTrue(run.published) + self.assertEqual( + dict(run.publication.provider_generations), + {"gateway": 2}, + ) + + +def compiled_registry_with_store(compiled_store, *adapters): + """Create a restarted registry against an existing compiled store.""" + registry = FragmentAdapterRegistry(enabled=True) + discovered = registry.discover([FragmentComponent(adapter) for adapter in adapters]) + compiler = registry.create_compiler(compiled_store) + return registry, compiler, compiled_store, discovered + + +if __name__ == "__main__": + unittest.main() From 33e03d23fc2e8be0618ed5095b87a7fbfbeab670 Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Wed, 5 Aug 2026 20:05:42 +0100 Subject: [PATCH 7/8] fix(lattice): complete minimal autoconfig core --- apps/predbat/lattice_fragment_adapters.py | 46 +- apps/predbat/lattice_topology.py | 410 ++++++++++++++++++ .../tests/test_lattice_fragment_adapters.py | 25 +- ..._lattice_fragment_tombstone_compilation.py | 258 +++++++++++ apps/predbat/tests/test_lattice_topology.py | 189 ++++++++ 5 files changed, 916 insertions(+), 12 deletions(-) create mode 100644 apps/predbat/lattice_topology.py create mode 100644 apps/predbat/tests/test_lattice_fragment_tombstone_compilation.py create mode 100644 apps/predbat/tests/test_lattice_topology.py diff --git a/apps/predbat/lattice_fragment_adapters.py b/apps/predbat/lattice_fragment_adapters.py index e8c4623d1..22f21e1d5 100644 --- a/apps/predbat/lattice_fragment_adapters.py +++ b/apps/predbat/lattice_fragment_adapters.py @@ -20,6 +20,7 @@ import hashlib import threading from dataclasses import dataclass +from functools import partial from types import MappingProxyType from typing import Optional, Protocol @@ -110,6 +111,43 @@ def __post_init__(self): object.__setattr__(self, "provider_id", provider_id) +def _compiler_fragment_snapshot(adapter): + """Fresh-read one compiler input, translating only durable removals.""" + state = adapter.read_state() + if not isinstance(state, FragmentAdapterState): + raise FragmentAdapterReadError("adapter read_state must return FragmentAdapterState") + try: + state.__post_init__() + except ValueError as exc: + raise FragmentAdapterReadError(str(exc)) from exc + provider_id = _validate_provider_id(getattr(adapter, "provider_id", None)) + if state.provider_id != provider_id: + raise FragmentAdapterReadError("adapter state belongs to provider {}".format(state.provider_id)) + if not state.removed: + return state.snapshot + return ProviderSnapshot( + provider_id=state.provider_id, + generation=state.generation, + health=ProviderHealth.HEALTHY, + topology_fragment={ + "topologyVersion": "0.3.0", + "scope": "fragment", + "docVersion": state.generation, + "producer": { + "name": "PredBat fragment tombstone", + "provider": state.provider_id, + "authority": 0, + }, + "nodes": [], + "relationships": [], + }, + aliases=(), + identity_aliases=(), + role_assignments=(), + config_projections=(), + ) + + class FragmentAdapterStateStore: """Required atomic durable-store protocol owned by one integration.""" @@ -484,7 +522,13 @@ def create_compiler(self, state_store, override_reader=None): raise RuntimeError("cannot create a compiler without fragment adapters") for adapter in self._adapters.values(): self._validate_adapter(adapter) - readers = {provider_id: adapter.read_snapshot for provider_id, adapter in self._adapters.items()} + readers = { + provider_id: partial( + _compiler_fragment_snapshot, + adapter, + ) + for provider_id, adapter in self._adapters.items() + } compiler = CompiledLatticeCompiler( readers, state_store=state_store, diff --git a/apps/predbat/lattice_topology.py b/apps/predbat/lattice_topology.py new file mode 100644 index 000000000..3fca096a5 --- /dev/null +++ b/apps/predbat/lattice_topology.py @@ -0,0 +1,410 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System - Lattice topology shadow store (read-only) +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +"""Read-only Lattice topology ingestion, merge, and source-reference provenance. + +The merged site document remints capability ``ref`` values. Those refs are not +safe to send to a provider: a control dispatcher must use the ref from the +provider-local source document. This module therefore keeps an out-of-band +sidecar keyed by ``(node id, capability, access path)``. + +There are deliberately no MQTT or Predbat dependencies here. Control +publishing, planner writes, and inverter writes are outside this module. +""" + +import copy +import json +from dataclasses import dataclass +from typing import Optional + + +SUPPORTED_TOPOLOGY_VERSIONS = frozenset(("0.2", "0.3")) + + +class TopologyValidationError(ValueError): + """Raised when an incoming topology document is unsafe to ingest.""" + + +@dataclass(frozen=True) +class SourceCapabilityRef: + """Provider-local coordinates for one winning capability offer.""" + + provider: str + topology_version: str + doc_version: int + cap_ref: int + node_id: str + capability: str + access_path: str + + +@dataclass(frozen=True) +class TopologySnapshot: + """An immutable-by-convention merged site and its provenance sidecar.""" + + site: dict + provenance: dict + warnings: tuple + + def source_ref(self, node_id, capability, access_path=None) -> Optional[SourceCapabilityRef]: + """Return the winning provider-local ref for a future dispatcher. + + When ``access_path`` is omitted, select the highest-preference merged + access path that offers the capability. Exact lookup remains available + so a dispatcher can follow a resolver's chosen fallback path. + """ + node_key = str(node_id) + capability_key = str(capability) + if access_path is not None: + return self.provenance.get((node_key, capability_key, str(access_path))) + + node = next((item for item in self.site.get("nodes", []) if str(item.get("id")) == node_key), None) + if node is None: + return None + preferences = {str(path.get("id")): path.get("preference", 0) or 0 for path in node.get("accessPaths", []) if path.get("id") is not None} + candidates = [] + for key, source in self.provenance.items(): + if key[0] == node_key and key[1] == capability_key: + candidates.append((-(preferences.get(key[2], 0)), key[2], source)) + candidates.sort(key=lambda item: (item[0], item[1])) + return candidates[0][2] if candidates else None + + +def _topology_family(version): + """Return the supported major/minor family for a topology version.""" + parts = str(version).split(".") + return ".".join(parts[:2]) if len(parts) >= 2 else str(version) + + +def _authority(document): + """Return producer authority, excluding bools from Python's integer type.""" + value = (document.get("producer") or {}).get("authority") + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def _doc_version(document): + """Return a validated document version, defaulting legacy 0.2 fragments to zero.""" + value = document.get("docVersion") + if value is None and _topology_family(document.get("topologyVersion")) == "0.2": + return 0 + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise TopologyValidationError("docVersion must be a non-negative integer") + return value + + +def _rank(document, order): + """Build the authority, recency, input-order rank tuple.""" + return (_authority(document), _doc_version(document), -order) + + +def _better(left, right): + """Return whether ``left`` outranks ``right``.""" + return left > right + + +def _winner(contributions): + """Return the highest-ranked contribution, preserving first on an exact tie.""" + best = contributions[0] + for contribution in contributions[1:]: + if _better(contribution[1], best[1]): + best = contribution + return best + + +def _validate_document(document): + """Validate the minimum shape needed for deterministic safe ingestion.""" + if not isinstance(document, dict): + raise TopologyValidationError("topology payload must be a JSON object") + version = document.get("topologyVersion") + if _topology_family(version) not in SUPPORTED_TOPOLOGY_VERSIONS: + raise TopologyValidationError("unsupported topologyVersion {!r}".format(version)) + if document.get("scope") not in ("fragment", "overlay", "site"): + raise TopologyValidationError("scope must be fragment, overlay, or site") + producer = document.get("producer") + if not isinstance(producer, dict) or not isinstance(producer.get("provider"), str) or not producer["provider"]: + raise TopologyValidationError("producer.provider must be a non-empty string") + _doc_version(document) + nodes = document.get("nodes", []) + if not isinstance(nodes, list): + raise TopologyValidationError("nodes must be an array") + for node in nodes: + if not isinstance(node, dict) or node.get("id") is None: + raise TopologyValidationError("every node must be an object with an id") + for field in ("accessPaths", "capabilities"): + if field in node and not isinstance(node[field], list): + raise TopologyValidationError("node {} {} must be an array".format(node["id"], field)) + return document + + +def decode_topology(payload): + """Decode and validate a retained MQTT topology payload.""" + if isinstance(payload, dict): + document = copy.deepcopy(payload) + else: + if isinstance(payload, bytes): + try: + payload = payload.decode("utf-8") + except UnicodeDecodeError as exc: + raise TopologyValidationError("topology payload is not UTF-8") from exc + if not isinstance(payload, str) or not payload.strip(): + raise TopologyValidationError("topology payload is empty") + try: + document = json.loads(payload) + except json.JSONDecodeError as exc: + raise TopologyValidationError("topology payload is malformed JSON") from exc + return _validate_document(document) + + +def _collection_winners(contributions, field, key_function): + """Merge an identity-keyed collection and retain each source document.""" + order = [] + values = {} + for item, rank, document in contributions: + entries = item.get(field, []) + if not isinstance(entries, list): + continue + for entry in entries: + if not isinstance(entry, dict): + continue + key = key_function(entry) + if key is None: + continue + if key not in values: + order.append(key) + current = values.get(key) + if current is None or _better(rank, current[1]): + values[key] = (entry, rank, document) + return [(key, values[key]) for key in order if values[key][0].get("removed") is not True] + + +def _offer_key(offer): + """Return a capability-offer identity.""" + if offer.get("capability") is None: + return None + return (str(offer["capability"]), str(offer.get("accessPath", ""))) + + +def _pick_field(contributions, field, node_id, warnings): + """Choose a scalar field by authority, recency, then input order.""" + setters = [(item[field], rank) for item, rank, _document in contributions if field in item] + if not setters: + return None + value, rank = _winner(setters) + for other_value, other_rank in setters: + if other_rank[:2] == rank[:2] and other_value != value: + warnings.append('node "{}" field "{}" tied; kept first input'.format(node_id, field)) + break + return copy.deepcopy(value) + + +def _merge_bag(contributions, field): + """Merge an attributes/parameters bag per key.""" + values = {} + for item, rank, _document in contributions: + bag = item.get(field) + if not isinstance(bag, dict): + continue + for key, value in bag.items(): + if key not in values or _better(rank, values[key][1]): + values[key] = (copy.deepcopy(value), rank) + return {key: value for key, (value, _rank_value) in values.items()} + + +def _source_ref(document, node_id, offer): + """Build source coordinates for a winning capability offer.""" + source_ref = offer.get("ref") + if not isinstance(source_ref, int) or isinstance(source_ref, bool) or source_ref <= 0: + return None + return SourceCapabilityRef( + provider=document["producer"]["provider"], + topology_version=document["topologyVersion"], + doc_version=_doc_version(document), + cap_ref=source_ref, + node_id=str(node_id), + capability=str(offer["capability"]), + access_path=str(offer.get("accessPath", "")), + ) + + +def _merge_node(contributions, warnings): + """Merge one surviving node and return its source-offer sidecar.""" + winner = _winner([(item, rank) for item, rank, _document in contributions])[0] + node_id = str(winner["id"]) + node = {"id": copy.deepcopy(winner["id"])} + for field in ("kind", "deviceType", "aggregate"): + value = _pick_field(contributions, field, node_id, warnings) + if value is not None: + node[field] = value + for field in ("attributes", "parameters"): + bag = _merge_bag(contributions, field) + if bag: + node[field] = bag + + path_winners = _collection_winners(contributions, "accessPaths", lambda path: str(path["id"]) if path.get("id") is not None else None) + paths = [copy.deepcopy(value[0]) for _key, value in path_winners] + paths.sort(key=lambda path: (-(path.get("preference", 0) or 0), str(path.get("id")))) + if paths: + node["accessPaths"] = paths + + offer_winners = _collection_winners(contributions, "capabilities", _offer_key) + offers = [] + provenance = {} + for key, (offer, _rank_value, document) in offer_winners: + merged_offer = {name: copy.deepcopy(value) for name, value in offer.items() if name not in ("removed", "ref")} + offers.append(merged_offer) + source = _source_ref(document, node_id, offer) + if source is not None: + provenance[(node_id, key[0], key[1])] = source + else: + warnings.append('node "{}" capability "{}" on "{}" has no provider-local ref'.format(node_id, key[0], key[1])) + if offers: + node["capabilities"] = offers + return node, provenance + + +def _digest(document): + """Return a deterministic positive 31-bit digest for merged docVersion.""" + encoded = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + value = 0x811C9DC5 + for character in encoded: + value ^= ord(character) + value = (value * 0x01000193) & 0xFFFFFFFF + return (value % 2147483647) + 1 + + +def merge_topologies(documents): + """Authority-merge topology documents while retaining local capability refs.""" + documents = [copy.deepcopy(_validate_document(document)) for document in documents] + if not documents: + raise TopologyValidationError("cannot merge an empty topology set") + + ranked_documents = [(document, _rank(document, order)) for order, document in enumerate(documents)] + top_document = _winner(ranked_documents)[0] + warnings = [] + + node_order = [] + node_contributions = {} + for document, rank in ranked_documents: + for node in document.get("nodes", []): + key = str(node["id"]) + if key not in node_contributions: + node_order.append(key) + node_contributions[key] = [] + node_contributions[key].append((node, rank, document)) + + nodes = [] + provenance = {} + surviving = set() + for node_id in node_order: + contributions = node_contributions[node_id] + if _winner([(item, rank) for item, rank, _document in contributions])[0].get("removed") is True: + continue + tombstones = [rank for item, rank, _document in contributions if item.get("removed") is True] + barrier = max(tombstones) if tombstones else None + live = [(item, rank, document) for item, rank, document in contributions if item.get("removed") is not True and (barrier is None or _better(rank, barrier) or rank == barrier)] + node, node_provenance = _merge_node(live, warnings) + nodes.append(node) + provenance.update(node_provenance) + surviving.add(node_id) + + relationship_contributions = {} + relationship_order = [] + for document, rank in ranked_documents: + for relationship in document.get("relationships", []): + if not isinstance(relationship, dict) or any(relationship.get(field) is None for field in ("from", "to", "type")): + continue + key = (str(relationship["from"]), str(relationship["to"]), str(relationship["type"])) + if key not in relationship_contributions: + relationship_order.append(key) + relationship_contributions[key] = [] + relationship_contributions[key].append((relationship, rank)) + relationships = [] + for key in relationship_order: + relationship = _winner(relationship_contributions[key])[0] + if relationship.get("removed") is True: + continue + if key[0] not in surviving or key[1] not in surviving: + warnings.append("relationship {} dropped because an endpoint is absent".format("|".join(key))) + continue + relationships.append({name: copy.deepcopy(value) for name, value in relationship.items() if name != "removed"}) + + next_ref = 1 + for node in nodes: + refs = {} + for offer in node.get("capabilities", []): + capability = str(offer.get("capability")) + if capability not in refs: + refs[capability] = next_ref + next_ref += 1 + offer["ref"] = refs[capability] + + site = { + "topologyVersion": top_document["topologyVersion"], + "scope": "site", + "producer": { + "name": "predbat-lattice-shadow", + "provider": "predbat", + "inputs": [ + { + "name": document.get("producer", {}).get("name"), + "provider": document["producer"]["provider"], + "authority": _authority(document), + "docVersion": _doc_version(document), + "topologyVersion": document["topologyVersion"], + } + for document, _rank_value in ranked_documents + ], + }, + "nodes": nodes, + } + if relationships: + site["relationships"] = relationships + site["docVersion"] = _digest(site) + return TopologySnapshot(site=site, provenance=provenance, warnings=tuple(warnings)) + + +class LatticeTopologyStore: + """Replace provider documents and expose the current merged shadow snapshot.""" + + def __init__(self): + """Create an empty provider document set.""" + self._documents = {} + self.snapshot = None + + def ingest(self, payload): + """Ingest a provider document, replacing only with a newer version. + + Returns ``True`` when the current snapshot changes. Stale or duplicate + documents are ignored. Invalid documents raise + :class:`TopologyValidationError` and leave the prior snapshot intact. + """ + document = decode_topology(payload) + provider = document["producer"]["provider"] + previous = self._documents.get(provider) + if previous is not None: + incoming_version = _doc_version(document) + previous_version = _doc_version(previous) + if incoming_version < previous_version: + return False + if incoming_version == previous_version: + if document == previous: + return False + # Current gateway 0.2 retained fragments predate docVersion. Arrival + # order is their only replacement signal; explicit versions remain + # protected against reuse with different content. + if "docVersion" in document or "docVersion" in previous: + raise TopologyValidationError("provider {} reused docVersion {} for different content".format(provider, incoming_version)) + candidate_documents = dict(self._documents) + candidate_documents[provider] = document + snapshot = merge_topologies(candidate_documents.values()) + self._documents = candidate_documents + self.snapshot = snapshot + return True + + def source_ref(self, node_id, capability, access_path=None) -> Optional[SourceCapabilityRef]: + """Expose provider-local capability coordinates for a future dispatcher.""" + if self.snapshot is None: + return None + return self.snapshot.source_ref(node_id, capability, access_path) diff --git a/apps/predbat/tests/test_lattice_fragment_adapters.py b/apps/predbat/tests/test_lattice_fragment_adapters.py index b0de1b4f0..adf075213 100644 --- a/apps/predbat/tests/test_lattice_fragment_adapters.py +++ b/apps/predbat/tests/test_lattice_fragment_adapters.py @@ -401,8 +401,8 @@ def test_other_provider_invalidation_exposes_reader_failure_fail_closed(self): {issue.code for issue in run.issues}, ) - def test_removal_invalidates_and_preserves_last_known_good(self): - """A durable removal triggers a bounded fail-closed recompile.""" + def test_removal_invalidates_and_clears_prior_provider_contribution(self): + """A durable removal compiles as an acknowledged empty fragment.""" adapter, _store, _initial = publisher("gateway") _registry, compiler, _compiled_store, _ids = compiled_registry(adapter) first = compiler.drain() @@ -411,13 +411,16 @@ def test_removal_invalidates_and_preserves_last_known_good(self): run = compiler.drain() self.assertEqual(run.attempts, 1) - self.assertEqual(run.status, CompileStatus.STALE) - self.assertTrue(run.pending) - self.assertIs(run.publication, first.publication) - self.assertIn( - "provider_read_failed", - {issue.code for issue in run.issues}, + self.assertEqual(run.status, CompileStatus.FRESH) + self.assertFalse(run.pending) + self.assertTrue(run.published) + self.assertEqual(run.publication.lattice_version, 2) + self.assertEqual( + dict(run.publication.provider_generations), + {"gateway": 2}, ) + self.assertEqual(run.plan.topology["nodes"], ()) + self.assertNotEqual(run.publication, first.publication) def test_restart_restores_adapter_and_compiler_cursor_protection(self): """Both durable layers reject reuse after a complete process restart.""" @@ -482,7 +485,8 @@ def test_invalidation_during_read_gets_one_bounded_follow_up(self): adapter, _store, initial = publisher("gateway") registry = FragmentAdapterRegistry(enabled=True) registry.register(adapter) - original_reader = adapter.read_snapshot + compiler = registry.create_compiler(InMemoryCompiledLatticeStateStore()) + original_reader = adapter.read_state fired = [False] def invalidating_reader(): @@ -496,8 +500,7 @@ def invalidating_reader(): ) return value - adapter.read_snapshot = invalidating_reader - compiler = registry.create_compiler(InMemoryCompiledLatticeStateStore()) + adapter.read_state = invalidating_reader run = compiler.drain() self.assertEqual(run.attempts, 2) diff --git a/apps/predbat/tests/test_lattice_fragment_tombstone_compilation.py b/apps/predbat/tests/test_lattice_fragment_tombstone_compilation.py new file mode 100644 index 000000000..cb251b76c --- /dev/null +++ b/apps/predbat/tests/test_lattice_fragment_tombstone_compilation.py @@ -0,0 +1,258 @@ +"""Focused coverage for compiler-only fragment tombstone translation.""" + +# cspell:ignore autoconfig + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from lattice_autoconfig import CompileStatus, ProviderHealth # noqa: E402 +from lattice_compiled_publication import ( # noqa: E402 + InMemoryCompiledLatticeStateStore, +) +from lattice_fragment_adapters import ( # noqa: E402 + DurableFragmentAdapter, + FragmentAdapterState, + FragmentAdapterRegistry, + FragmentAdapterRemoved, + InMemoryFragmentAdapterStateStore, + _compiler_fragment_snapshot, +) +from tests.test_lattice_autoconfig import snapshot # noqa: E402 + + +def publisher(provider_id, node_id): + """Create one seeded integration-owned fragment publisher.""" + store = InMemoryFragmentAdapterStateStore() + adapter = DurableFragmentAdapter(provider_id, store) + initial = snapshot( + provider_id, + generation=1, + node_id=node_id, + ) + adapter.publish(initial, "initial fragment") + return adapter, store, initial + + +def compiler_for(store, *adapters): + """Create one explicitly enabled compiler over fixed membership.""" + registry = FragmentAdapterRegistry(enabled=True) + for adapter in adapters: + registry.register(adapter) + return registry, registry.create_compiler(store) + + +def structurally_corrupt_state( + provider_id, + generation, + semantic_fingerprint, + provider_snapshot, + removed=False, +): + """Bypass construction validation to model a corrupt structural reader.""" + state = object.__new__(FragmentAdapterState) + object.__setattr__(state, "provider_id", provider_id) + object.__setattr__(state, "generation", generation) + object.__setattr__( + state, + "semantic_fingerprint", + semantic_fingerprint, + ) + object.__setattr__(state, "snapshot", provider_snapshot) + object.__setattr__(state, "removed", removed) + return state + + +class TestCompilerFragmentTombstones(unittest.TestCase): + """Removal is empty compiler input without weakening adapter reads.""" + + def test_live_snapshot_is_exact_and_removed_snapshot_is_empty(self): + """Translation changes only a durable removal state.""" + adapter, _store, initial = publisher("gateway", "GW-INV") + + self.assertIs(_compiler_fragment_snapshot(adapter), initial) + self.assertTrue(adapter.remove(2, "integration removed")) + + with self.assertRaises(FragmentAdapterRemoved): + adapter.read_snapshot() + tombstone = _compiler_fragment_snapshot(adapter) + + self.assertEqual(tombstone.provider_id, "gateway") + self.assertEqual(tombstone.generation, 2) + self.assertEqual(tombstone.health, ProviderHealth.HEALTHY) + self.assertEqual(tombstone.topology_fragment["nodes"], ()) + self.assertEqual(tombstone.topology_fragment["relationships"], ()) + self.assertEqual(tombstone.aliases, ()) + self.assertEqual(tombstone.identity_aliases, ()) + self.assertEqual(tombstone.role_assignments, ()) + self.assertEqual(tombstone.config_projections, ()) + + def test_all_removed_publishes_deterministic_empty_plan_and_restarts(self): + """All tombstones settle, persist, and restore as one empty plan.""" + alpha, alpha_store, _alpha = publisher("alpha", "ALPHA-INV") + beta, beta_store, _beta = publisher("beta", "BETA-INV") + compiled_store = InMemoryCompiledLatticeStateStore() + _registry, compiler = compiler_for(compiled_store, alpha, beta) + baseline = compiler.drain() + + self.assertTrue(beta.remove(2, "beta removed")) + self.assertTrue(alpha.remove(2, "alpha removed")) + removed = compiler.drain() + + self.assertEqual(removed.status, CompileStatus.FRESH) + self.assertTrue(removed.published) + self.assertFalse(removed.pending) + self.assertEqual(removed.plan.topology["nodes"], ()) + self.assertEqual(removed.plan.aliases, ()) + self.assertEqual( + dict(removed.publication.provider_generations), + {"alpha": 2, "beta": 2}, + ) + self.assertEqual( + dict(removed.publication.provider_requested_generations), + {"alpha": 2, "beta": 2}, + ) + self.assertNotEqual( + removed.publication.digest, + baseline.publication.digest, + ) + + restarted_alpha = DurableFragmentAdapter("alpha", alpha_store) + restarted_beta = DurableFragmentAdapter("beta", beta_store) + _registry, restarted = compiler_for( + compiled_store, + restarted_beta, + restarted_alpha, + ) + settled = restarted.drain() + + self.assertFalse(settled.published) + self.assertFalse(settled.pending) + self.assertEqual(settled.publication, removed.publication) + self.assertEqual(settled.plan.digest, removed.plan.digest) + + def test_removed_payload_cannot_reappear_at_the_same_generation(self): + """A tombstone never leaks its payload and cursor reuse remains unsafe.""" + adapter, _store, initial = publisher("cloud", "CLOUD-INV") + self.assertTrue(adapter.remove(2, "cloud removed")) + + empty = _compiler_fragment_snapshot(adapter) + self.assertEqual(empty.topology_fragment["nodes"], ()) + with self.assertRaisesRegex(ValueError, "reused"): + adapter.publish( + snapshot( + "cloud", + generation=2, + node_id=initial.topology_fragment["nodes"][0]["id"], + ), + "attempted resurrection", + ) + self.assertEqual( + _compiler_fragment_snapshot(adapter).topology_fragment["nodes"], + (), + ) + + def test_fresh_read_revalidates_semantic_fingerprint_and_keeps_lkg(self): + """Post-registration state corruption cannot reach publication.""" + adapter, _store, _initial = publisher("gateway", "GW-INV") + compiled_store = InMemoryCompiledLatticeStateStore() + _registry, compiler = compiler_for(compiled_store, adapter) + baseline = compiler.drain() + corrupt = structurally_corrupt_state( + "gateway", + 2, + "0" * 64, + snapshot( + "gateway", + generation=2, + node_id="GW-INV", + ), + ) + adapter.read_state = lambda: corrupt + + self.assertTrue( + compiler.invalidate( + "gateway", + 2, + "corrupt structural read", + ) + ) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertTrue(failed.pending) + self.assertFalse(failed.published) + self.assertIs(failed.publication, baseline.publication) + self.assertEqual(compiled_store.writes, 1) + self.assertIn( + "provider_read_failed", + {issue.code for issue in failed.issues}, + ) + + def test_fresh_read_revalidates_snapshot_cursor_bindings(self): + """Provider and generation binding corruption both fail closed.""" + cases = ( + ( + "provider", + snapshot( + "other-provider", + generation=2, + node_id="GW-INV", + ), + ), + ( + "generation", + snapshot( + "gateway", + generation=3, + node_id="GW-INV", + ), + ), + ) + for label, corrupt_snapshot in cases: + with self.subTest(binding=label): + adapter, _store, _initial = publisher( + "gateway", + "GW-INV", + ) + compiled_store = InMemoryCompiledLatticeStateStore() + _registry, compiler = compiler_for( + compiled_store, + adapter, + ) + baseline = compiler.drain() + corrupt = structurally_corrupt_state( + "gateway", + 2, + adapter.read_state().semantic_fingerprint, + corrupt_snapshot, + ) + adapter.read_state = lambda value=corrupt: value + + self.assertTrue( + compiler.invalidate( + "gateway", + 2, + "{} binding corruption".format(label), + ) + ) + failed = compiler.drain() + + self.assertEqual(failed.status, CompileStatus.STALE) + self.assertTrue(failed.pending) + self.assertFalse(failed.published) + self.assertIs( + failed.publication, + baseline.publication, + ) + self.assertEqual(compiled_store.writes, 1) + self.assertIn( + "provider_read_failed", + {issue.code for issue in failed.issues}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/apps/predbat/tests/test_lattice_topology.py b/apps/predbat/tests/test_lattice_topology.py new file mode 100644 index 000000000..577042aaa --- /dev/null +++ b/apps/predbat/tests/test_lattice_topology.py @@ -0,0 +1,189 @@ +"""Tests for read-only Lattice topology ingestion and provenance.""" + +import ast +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from lattice_topology import LatticeTopologyStore, TopologyValidationError, merge_topologies + + +def topology(provider, doc_version, authority=0, topology_version="0.3.0", kind="inverter", capability="battery.target_soc", access_path=None, cap_ref=1): + """Build a compact provider-local topology document.""" + access_path = access_path or "{}-local".format(provider) + return { + "topologyVersion": topology_version, + "scope": "fragment", + "docVersion": doc_version, + "producer": {"name": provider, "provider": provider, "authority": authority}, + "nodes": [ + { + "id": "INV1", + "kind": kind, + "accessPaths": [{"id": access_path, "provider": provider, "preference": authority}], + "capabilities": [ + { + "capability": capability, + "accessPath": access_path, + "ref": cap_ref, + "shape": "setpoint", + "control": {"protocol": "mqtt"}, + } + ], + } + ], + } + + +class TestTopologyReplacement(unittest.TestCase): + """Provider documents replace atomically by docVersion.""" + + def test_newer_replaces_and_stale_is_ignored(self): + """A newer retained provider document replaces its predecessor.""" + store = LatticeTopologyStore() + self.assertTrue(store.ingest(topology("gateway", 1, cap_ref=11))) + first_doc_version = store.snapshot.site["docVersion"] + self.assertTrue(store.ingest(topology("gateway", 2, cap_ref=12))) + self.assertEqual(store.source_ref("INV1", "battery.target_soc").cap_ref, 12) + self.assertNotEqual(store.snapshot.site["docVersion"], first_doc_version) + self.assertFalse(store.ingest(topology("gateway", 1, cap_ref=99))) + self.assertEqual(store.source_ref("INV1", "battery.target_soc").cap_ref, 12) + + def test_duplicate_is_ignored_and_reused_version_is_rejected(self): + """A docVersion cannot name two different documents from one provider.""" + store = LatticeTopologyStore() + document = topology("gateway", 4, cap_ref=10) + self.assertTrue(store.ingest(document)) + self.assertFalse(store.ingest(json.dumps(document).encode())) + with self.assertRaises(TopologyValidationError): + store.ingest(topology("gateway", 4, cap_ref=20)) + + def test_malformed_document_preserves_previous_snapshot(self): + """Malformed retained data cannot erase the last known-good topology.""" + store = LatticeTopologyStore() + store.ingest(topology("gateway", 1, cap_ref=11)) + before = store.snapshot + with self.assertRaises(TopologyValidationError): + store.ingest(b"{broken") + self.assertIs(store.snapshot, before) + + def test_gateway_02_without_doc_version_replaces_by_arrival(self): + """Current gateway 0.2 fragments work before firmware adds docVersion.""" + store = LatticeTopologyStore() + first = topology("predbat-gateway", 1, topology_version="0.2.0", cap_ref=11) + second = topology("predbat-gateway", 1, topology_version="0.2.0", cap_ref=12) + del first["docVersion"] + del second["docVersion"] + self.assertTrue(store.ingest(first)) + self.assertTrue(store.ingest(second)) + source = store.source_ref("INV1", "battery.target_soc") + self.assertEqual(source.doc_version, 0) + self.assertEqual(source.cap_ref, 12) + + +class TestAuthorityMerge(unittest.TestCase): + """Authority, recency, and stable input order choose deterministic winners.""" + + def test_authority_then_recency_select_winner(self): + """Higher authority wins before docVersion; recency breaks authority ties.""" + low_new = topology("cloud", 99, authority=1, kind="battery", cap_ref=90) + high_old = topology("gateway", 1, authority=10, kind="inverter", cap_ref=10) + result = merge_topologies([low_new, high_old]) + self.assertEqual(result.site["nodes"][0]["kind"], "inverter") + + newer = topology("installer", 2, authority=10, kind="hybrid-inverter", access_path="installer", cap_ref=20) + result = merge_topologies([high_old, newer]) + self.assertEqual(result.site["nodes"][0]["kind"], "hybrid-inverter") + + def test_exact_tie_keeps_first_and_warns(self): + """Input order is the stable final tiebreak for scalar conflicts.""" + first = topology("a", 1, authority=5, kind="inverter", cap_ref=1) + second = topology("b", 1, authority=5, kind="battery", cap_ref=2) + result = merge_topologies([first, second]) + self.assertEqual(result.site["nodes"][0]["kind"], "inverter") + self.assertTrue(any("tied" in warning for warning in result.warnings)) + + def test_reminted_refs_map_to_provider_local_refs(self): + """Merged refs are independent while the sidecar retains each local ref.""" + gateway = topology("gateway", 7, authority=10, access_path="gw", cap_ref=41) + cloud = topology("cloud", 3, authority=1, access_path="cloud", cap_ref=900) + result = merge_topologies([gateway, cloud]) + offers = result.site["nodes"][0]["capabilities"] + self.assertEqual({offer["ref"] for offer in offers}, {1}) + self.assertEqual(result.source_ref("INV1", "battery.target_soc", "gw").cap_ref, 41) + self.assertEqual(result.source_ref("INV1", "battery.target_soc", "cloud").cap_ref, 900) + self.assertEqual(result.source_ref("INV1", "battery.target_soc").provider, "gateway") + + def test_mixed_02_and_03_providers_retain_source_versions(self): + """Compatible 0.x providers keep their individual topology versions.""" + old = topology("cloud", 1, authority=1, topology_version="0.2.0", access_path="cloud", cap_ref=2) + current = topology("gateway", 2, authority=10, topology_version="0.3.0", access_path="gw", cap_ref=3) + result = merge_topologies([old, current]) + self.assertEqual(result.site["topologyVersion"], "0.3.0") + self.assertEqual(result.source_ref("INV1", "battery.target_soc", "cloud").topology_version, "0.2.0") + self.assertEqual(result.source_ref("INV1", "battery.target_soc", "gw").topology_version, "0.3.0") + + def test_capability_and_node_tombstones_remove_provenance(self): + """Winning tombstones remove merged offers/nodes and their sidecar entries.""" + discovered = topology("gateway", 1, authority=0, access_path="gw", cap_ref=3) + capability_tombstone = { + "topologyVersion": "0.3.0", + "scope": "overlay", + "docVersion": 1, + "producer": {"name": "installer", "provider": "installer", "authority": 50}, + "nodes": [{"id": "INV1", "capabilities": [{"capability": "battery.target_soc", "accessPath": "gw", "removed": True}]}], + } + result = merge_topologies([discovered, capability_tombstone]) + self.assertNotIn("capabilities", result.site["nodes"][0]) + self.assertIsNone(result.source_ref("INV1", "battery.target_soc", "gw")) + + node_tombstone = { + "topologyVersion": "0.3.0", + "scope": "overlay", + "docVersion": 2, + "producer": {"name": "installer", "provider": "installer", "authority": 50}, + "nodes": [{"id": "INV1", "removed": True}], + } + result = merge_topologies([discovered, node_tombstone]) + self.assertEqual(result.site["nodes"], []) + self.assertEqual(result.provenance, {}) + + def test_node_tombstone_exact_tie_respects_stable_input_order(self): + """Equal-rank live/tombstone conflicts keep the first contribution.""" + live = topology("gateway", 1, authority=10, cap_ref=3) + tombstone = { + "topologyVersion": "0.3.0", + "scope": "overlay", + "docVersion": 1, + "producer": {"name": "installer", "provider": "installer", "authority": 10}, + "nodes": [{"id": "INV1", "removed": True}], + } + + live_first = merge_topologies([live, tombstone]) + self.assertEqual([node["id"] for node in live_first.site["nodes"]], ["INV1"]) + self.assertEqual(live_first.source_ref("INV1", "battery.target_soc").cap_ref, 3) + + tombstone_first = merge_topologies([tombstone, live]) + self.assertEqual(tombstone_first.site["nodes"], []) + self.assertEqual(tombstone_first.provenance, {}) + + +class TestGatewayFeatureFlag(unittest.TestCase): + """Gateway topology behavior is absent unless explicitly enabled.""" + + def test_feature_flag_defaults_off(self): + """Default initialization does not expose a dispatcher binding.""" + gateway_path = os.path.join(os.path.dirname(__file__), "..", "gateway.py") + with open(gateway_path, "r", encoding="utf-8") as gateway_file: + module = ast.parse(gateway_file.read()) + gateway_class = next(node for node in module.body if isinstance(node, ast.ClassDef) and node.name == "GatewayMQTT") + initialize = next(node for node in gateway_class.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "initialize") + defaults = dict(zip([argument.arg for argument in initialize.args.args[-len(initialize.args.defaults) :]], initialize.args.defaults)) + self.assertIs(defaults["lattice_projection_enable"].value, False) + + +if __name__ == "__main__": + unittest.main() From 5a61b929241c64f87540dedc47aba32c2bcd8528 Mon Sep 17 00:00:00 2001 From: Mark Gascoyne Date: Wed, 5 Aug 2026 20:07:12 +0100 Subject: [PATCH 8/8] test(lattice): keep topology core-only --- apps/predbat/tests/test_lattice_topology.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/apps/predbat/tests/test_lattice_topology.py b/apps/predbat/tests/test_lattice_topology.py index 577042aaa..5f8110e6d 100644 --- a/apps/predbat/tests/test_lattice_topology.py +++ b/apps/predbat/tests/test_lattice_topology.py @@ -1,6 +1,5 @@ """Tests for read-only Lattice topology ingestion and provenance.""" -import ast import json import os import sys @@ -171,19 +170,5 @@ def test_node_tombstone_exact_tie_respects_stable_input_order(self): self.assertEqual(tombstone_first.provenance, {}) -class TestGatewayFeatureFlag(unittest.TestCase): - """Gateway topology behavior is absent unless explicitly enabled.""" - - def test_feature_flag_defaults_off(self): - """Default initialization does not expose a dispatcher binding.""" - gateway_path = os.path.join(os.path.dirname(__file__), "..", "gateway.py") - with open(gateway_path, "r", encoding="utf-8") as gateway_file: - module = ast.parse(gateway_file.read()) - gateway_class = next(node for node in module.body if isinstance(node, ast.ClassDef) and node.name == "GatewayMQTT") - initialize = next(node for node in gateway_class.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "initialize") - defaults = dict(zip([argument.arg for argument in initialize.args.args[-len(initialize.args.defaults) :]], initialize.args.defaults)) - self.assertIs(defaults["lattice_projection_enable"].value, False) - - if __name__ == "__main__": unittest.main()