From 07bf76c752a049fa23e6c300a065624b145e0015 Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Thu, 23 Jul 2026 16:58:52 +0200 Subject: [PATCH] OpenTender: identifier-first dispatch (fetch_by_registration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add fetch_by_registration() to OpenTenderAdapter so the source can be queried by the national registration number the lookup pipeline already derives from the GLEIF anchor (siren / cz_ico / ee_registry_code / fi_business_id …) instead of a name-keyed FTS MATCH. This is the fix for the core of issue #29: an FTS MATCH on "Orange" returns genuine but unrelated bodies ("Red-Orange e.U.", "Orange controls s.r.o."), because relevance ranking is not identity. Keying on the indexed body_ids table (id_value, scoped to the subject's country, restricted to the id types that carry registration numbers — ORGANIZATION_ID / TRADE_REGISTER / HEADER_ICO / TAX_ID, excluding the internal SOURCE_ID / BVD_ID / ETALON_ID keys) returns only tenders the same legal entity actually took part in — the precise inverse of the name search's behaviour. The registration query mirrors the search/fetch corrupt-DB hardening (quick_check defence in depth: a DatabaseError drops the connection and returns empty rather than raising up the pipeline) and normalises the id into exact / digits-only / leading-zero-stripped forms (mirrors eiti._norm_forms) to bridge GLEIF/DIGIWHIST formatting drift. Tests pin the #29 regressions: the Orange-SIREN identifier hit, the "Orange ≠ Red-Orange e.U." name-collision rejection, country scoping, internal-id-type exclusion, id-form normalisation, and graceful empty degradation (no DB / blank inputs / corrupt DB). Scope: adapter method + tests only. Wiring into REGISTRY / the lookup dispatch is deliberately deferred — the issue notes registration is separately blocked on artifact size (Render cold-start budget), and it is a dispatch-framework change outside this change's scope. Co-Authored-By: Claude Opus 4.8 --- backend/opencheck/sources/opentender.py | 136 +++++++++++++ backend/tests/test_opentender.py | 251 +++++++++++++++++++++++- 2 files changed, 378 insertions(+), 9 deletions(-) diff --git a/backend/opencheck/sources/opentender.py b/backend/opencheck/sources/opentender.py index 0a9c7b8..b9da880 100644 --- a/backend/opencheck/sources/opentender.py +++ b/backend/opencheck/sources/opentender.py @@ -284,6 +284,106 @@ def _db_search_impl(self, conn: sqlite3.Connection, query: str) -> list[SourceHi hits.append(self._tender_hit(tender)) return hits + # ------------------------------------------------------------------ + # Identifier-first dispatch (called by the lookup pipeline) + # ------------------------------------------------------------------ + + async def fetch_by_registration( + self, country: str, registration_number: str, legal_name: str = "" + ) -> list[SourceHit]: + """Return tenders in which the *identified* body took part. + + This is the identifier-first path (issue #29): rather than an FTS + ``MATCH`` over body names — which conflates "Orange" with the genuine + but unrelated bodies "Red-Orange e.U." and "Orange controls s.r.o." — + it keys on the national registration number the pipeline already + derives from the GLEIF anchor (``siren`` / ``cz_ico`` / ``ee_registry_code`` + / ``fi_business_id`` …). Keying on Orange S.A.'s SIREN returns only + Orange's telecoms contracts; keying on EDF's SIREN returns only EDF's + energy contracts — the precise inverse of the name search's noise. + + The query hits the indexed ``body_ids`` table on ``id_value``, restricted + to the id types that genuinely carry registration numbers + (``ORGANIZATION_ID`` / ``TRADE_REGISTER`` / ``HEADER_ICO`` / ``TAX_ID``); + the internal keys (``SOURCE_ID`` / ``BVD_ID`` / ``ETALON_ID``) are excluded. + It is scoped to the subject's ``country`` (already stored ISO-normalised + in ``tenders.country`` — DIGIWHIST ``UK`` is persisted as ``GB``) so a + registration number cannot collide across national registries. + + Returns an empty list in demo/stub mode (no DB), when the identifier or + country is absent, or when nothing matches — identifier dispatch has no + relevance fallback, by design. ``legal_name`` is accepted for parity + with the other ``fetch_by_registration`` adapters and is unused: the + identifier *is* the identity match, so no name gate is applied (a name + filter would wrongly drop a subsidiary whose registered name differs). + """ + reg = (registration_number or "").strip() + cc = (country or "").strip().upper() + if not reg or not cc: + return [] + + conn = self._conn() + if conn is not None: + return await asyncio.to_thread( + self._db_fetch_by_registration, conn, cc, reg + ) + # No live DB (demo/stub mode): identifier dispatch yields nothing — + # there is no fixture cache for the registration path. + return [] + + def _db_fetch_by_registration( + self, conn: sqlite3.Connection, country: str, registration_number: str + ) -> list[SourceHit]: + """Registration-keyed lookup, hardened against a corrupt DB that slipped + past the connect-time integrity check (defence in depth).""" + try: + return self._db_fetch_by_registration_impl( + conn, country, registration_number + ) + except sqlite3.DatabaseError as exc: + logger.error( + "opentender: DB error during registration lookup (%s/%s) — " + "returning no results and dropping the connection so it " + "revalidates next request: %s", + country, registration_number, exc, + ) + self._db = None + return [] + + def _db_fetch_by_registration_impl( + self, conn: sqlite3.Connection, country: str, registration_number: str + ) -> list[SourceHit]: + """Join ``body_ids`` (registration id types only) back to ``tenders``, + scoped by country, on any normalised form of the registration number.""" + forms = _id_forms(registration_number) + if not forms: + return [] + + value_ph = ",".join("?" * len(forms)) + type_ph = ",".join("?" * len(_REGISTRATION_ID_TYPES)) + cur = conn.execute( + f""" + SELECT DISTINCT t.persistent_id, t.data + FROM body_ids b + JOIN tenders t ON t.persistent_id = b.persistent_id + WHERE b.id_value IN ({value_ph}) + AND b.id_type IN ({type_ph}) + AND t.country = ? + LIMIT 20 + """, + (*forms, *_REGISTRATION_ID_TYPES, country), + ) + rows = cur.fetchall() + + hits: list[SourceHit] = [] + for row in rows: + try: + tender = json.loads(row["data"]) + except (json.JSONDecodeError, TypeError): + continue + hits.append(self._tender_hit(tender)) + return hits + # ------------------------------------------------------------------ # Fetch # ------------------------------------------------------------------ @@ -507,6 +607,42 @@ def warm_opentender_db() -> None: _LEI_SHAPE = re.compile(r"^[A-Z0-9]{20}$") +#: DIGIWHIST ``body_ids.id_type`` values that genuinely carry a *national +#: registration number* — the key a GLEIF-derived local id can match. Excludes +#: ``SOURCE_ID`` / ``BVD_ID`` / ``ETALON_ID`` (internal DIGIWHIST / Bureau van +#: Dijk house keys) and ``VAT`` (near-absent in the artifact — 6 rows — and a +#: distinct scheme). ``ORGANIZATION_ID`` is the national registration number. +_REGISTRATION_ID_TYPES: tuple[str, ...] = ( + "ORGANIZATION_ID", + "TRADE_REGISTER", + "HEADER_ICO", + "TAX_ID", +) + +_ID_DIGITS_RE = re.compile(r"\D+") + + +def _id_forms(value: str) -> list[str]: + """Candidate comparison forms for a national registration number. + + DIGIWHIST stores identifiers verbatim while GLEIF may publish the same + number spaced, punctuated or zero-padded. Exact string first, then + digits-only, then leading-zero-stripped — mirrors ``eiti._norm_forms`` so + ``0056.58.214`` matches ``005658214`` and ``01285743`` matches ``1285743``. + """ + value = (value or "").strip() + if not value: + return [] + forms = [value] + digits = _ID_DIGITS_RE.sub("", value) + if digits and digits not in forms: + forms.append(digits) + if digits: + stripped = digits.lstrip("0") + if stripped and stripped not in forms: + forms.append(stripped) + return forms + def _walk_bodies(tender: dict[str, Any]): """Yield every Body referenced by a DIGIWHIST tender. diff --git a/backend/tests/test_opentender.py b/backend/tests/test_opentender.py index 41eb734..ac5c5cc 100644 --- a/backend/tests/test_opentender.py +++ b/backend/tests/test_opentender.py @@ -17,6 +17,7 @@ from opencheck.sources.opentender import ( OpenTenderAdapter, _bridge_identifier, + _id_forms, _slug, ) @@ -92,23 +93,32 @@ def _make_db(tmp_path: Path, tenders: list[dict]) -> Path: json.dumps(tender), ), ) - # FTS entries for buyers + bidders. - for body in tender.get("buyers") or []: + # FTS + body_ids entries for buyers + bidders (mirrors the extract + # script: names go to the FTS index, every bodyIds[] entry goes to the + # flat body_ids identifier index with its raw id_value). + def _index_body(body: dict, role: str) -> None: name = (body.get("name") or "").strip() if name: cur.execute( "INSERT INTO body_names_fts (persistent_id, name, role) VALUES (?,?,?)", - (pid, name, "buyer"), + (pid, name, role), + ) + for ident in body.get("bodyIds") or []: + id_value = str(ident.get("id") or "").strip() + if not id_value: + continue + cur.execute( + "INSERT INTO body_ids (persistent_id, id_type, id_scope, id_value) " + "VALUES (?,?,?,?)", + (pid, str(ident.get("type") or ""), str(ident.get("scope") or ""), id_value), ) + + for body in tender.get("buyers") or []: + _index_body(body, "buyer") for lot in tender.get("lots") or []: for bid in lot.get("bids") or []: for body in bid.get("bidders") or []: - name = (body.get("name") or "").strip() - if name: - cur.execute( - "INSERT INTO body_names_fts (persistent_id, name, role) VALUES (?,?,?)", - (pid, name, "bidder"), - ) + _index_body(body, "bidder") conn.commit() conn.close() return db_path @@ -443,6 +453,229 @@ async def test_db_search_no_results_returns_empty(monkeypatch, tmp_path: Path) - assert hits == [] +# --------------------------------------------------------------------- +# Identifier-first dispatch — fetch_by_registration (issue #29) +# --------------------------------------------------------------------- + +# Orange S.A. — SIREN 380129866 — a genuine telecoms supplier the derived +# national ID resolves to. The lot bidder carries the SIREN as ORGANIZATION_ID. +_ORANGE_TENDER = { + "id": "cf-fr-orange", + "persistentId": "FR_orange_telecom_1", + "title": "FOURNITURE DE SERVICE DE TELECOMMUNICATIONS", + "country": "FR", + "isAwarded": True, + "buyers": [{"name": "Ministère de l'Intérieur", "bodyIds": []}], + "lots": [ + { + "bids": [ + { + "isWinning": True, + "bidders": [ + { + "name": "Orange S.A.", + "bodyIds": [ + {"id": "380129866", "type": "ORGANIZATION_ID", "scope": "FR"} + ], + } + ], + } + ] + } + ], +} + +# Red-Orange e.U. — an Austrian furniture supplier that a name-keyed FTS MATCH +# on "Orange" wrongly surfaces (issue #29). Different entity, different country, +# different registration number — identifier dispatch must NOT return it. +_RED_ORANGE_TENDER = { + "id": "cf-at-redorange", + "persistentId": "AT_red_orange_furniture_1", + "title": "Büromöbel Rahmenvereinbarung", + "country": "AT", + "isAwarded": True, + "buyers": [{"name": "Stadt Wien", "bodyIds": []}], + "lots": [ + { + "bids": [ + { + "isWinning": True, + "bidders": [ + { + "name": "Red-Orange e.U.", + "bodyIds": [ + {"id": "999888777", "type": "ORGANIZATION_ID", "scope": "AT"} + ], + } + ], + } + ] + } + ], +} + + +def _live_adapter(monkeypatch, tmp_path: Path, tenders: list[dict]) -> OpenTenderAdapter: + db_path = _make_db(tmp_path, tenders) + monkeypatch.setenv("OPENTENDER_DB_FILE", str(db_path)) + get_settings.cache_clear() + return OpenTenderAdapter() + + +async def test_fetch_by_registration_matches_by_national_id( + monkeypatch, tmp_path: Path +) -> None: + """Keying on Orange S.A.'s SIREN returns Orange's telecoms tender — the + identifier-first inverse of the name search.""" + adapter = _live_adapter(monkeypatch, tmp_path, [_ORANGE_TENDER, _RED_ORANGE_TENDER]) + hits = await adapter.fetch_by_registration("FR", "380129866", legal_name="Orange S.A.") + assert len(hits) == 1 + assert hits[0].hit_id == "FR_orange_telecom_1" + assert hits[0].name == "FOURNITURE DE SERVICE DE TELECOMMUNICATIONS" + + +async def test_fetch_by_registration_rejects_name_token_collision( + monkeypatch, tmp_path: Path +) -> None: + """The #29 regression: a name MATCH on 'Orange' surfaces the unrelated + Austrian 'Red-Orange e.U.', but the identifier path keyed on Orange's SIREN + never returns it.""" + adapter = _live_adapter(monkeypatch, tmp_path, [_ORANGE_TENDER, _RED_ORANGE_TENDER]) + + # Name search is the buggy path: 'Orange' pulls in Red-Orange e.U. as noise. + name_hits = await adapter.search("Orange", SearchKind.ENTITY) + name_ids = {h.hit_id for h in name_hits} + assert "AT_red_orange_furniture_1" in name_ids # the false positive + + # Identifier dispatch keyed on Orange's SIREN excludes the collision. + id_hits = await adapter.fetch_by_registration("FR", "380129866") + id_ids = {h.hit_id for h in id_hits} + assert id_ids == {"FR_orange_telecom_1"} + assert "AT_red_orange_furniture_1" not in id_ids + + +async def test_fetch_by_registration_is_country_scoped( + monkeypatch, tmp_path: Path +) -> None: + """The same registration number in a different country is not returned — + scoping prevents cross-registry id collisions.""" + adapter = _live_adapter(monkeypatch, tmp_path, [_ORANGE_TENDER]) + # Right number, wrong country. + assert await adapter.fetch_by_registration("CZ", "380129866") == [] + + +async def test_fetch_by_registration_ignores_internal_id_types( + monkeypatch, tmp_path: Path +) -> None: + """A body that carries the value only under an internal key (SOURCE_ID / + BVD_ID / ETALON_ID) is not a registration-number match.""" + tender = { + "id": "cf-fr-internal", + "persistentId": "FR_internal_only", + "title": "Internal-keyed tender", + "country": "FR", + "lots": [ + { + "bids": [ + { + "isWinning": True, + "bidders": [ + { + "name": "Some Supplier", + "bodyIds": [ + {"id": "552081317", "type": "SOURCE_ID", "scope": "FR"}, + {"id": "552081317", "type": "BVD_ID", "scope": "FR"}, + ], + } + ], + } + ] + } + ], + } + adapter = _live_adapter(monkeypatch, tmp_path, [tender]) + assert await adapter.fetch_by_registration("FR", "552081317") == [] + + +async def test_fetch_by_registration_normalises_id_forms( + monkeypatch, tmp_path: Path +) -> None: + """A punctuated / zero-padded GLEIF registeredAs matches the raw DIGIWHIST + id_value stored verbatim.""" + tender = { + "id": "cf-cz-ico", + "persistentId": "CZ_ico_1", + "title": "Czech works contract", + "country": "CZ", + "lots": [ + { + "bids": [ + { + "isWinning": True, + "bidders": [ + { + "name": "Stavby s.r.o.", + "bodyIds": [ + {"id": "45274649", "type": "HEADER_ICO", "scope": "CZ"} + ], + } + ], + } + ] + } + ], + } + adapter = _live_adapter(monkeypatch, tmp_path, [tender]) + # GLEIF might publish it zero-padded / spaced; both normalise to the raw form. + hits = await adapter.fetch_by_registration("CZ", "0045274649") + assert {h.hit_id for h in hits} == {"CZ_ico_1"} + + +async def test_fetch_by_registration_empty_without_db() -> None: + """Demo/stub mode (no DB) yields nothing — identifier dispatch has no + relevance fallback.""" + adapter = OpenTenderAdapter() + assert await adapter.fetch_by_registration("FR", "380129866") == [] + + +async def test_fetch_by_registration_empty_on_blank_inputs( + monkeypatch, tmp_path: Path +) -> None: + adapter = _live_adapter(monkeypatch, tmp_path, [_ORANGE_TENDER]) + assert await adapter.fetch_by_registration("FR", "") == [] + assert await adapter.fetch_by_registration("", "380129866") == [] + + +async def test_fetch_by_registration_no_match_returns_empty( + monkeypatch, tmp_path: Path +) -> None: + adapter = _live_adapter(monkeypatch, tmp_path, [_ORANGE_TENDER]) + assert await adapter.fetch_by_registration("FR", "000000000") == [] + + +async def test_fetch_by_registration_degrades_to_empty_when_db_corrupt( + monkeypatch, tmp_path: Path +) -> None: + """A malformed DB must degrade gracefully (empty), never raise + 'database disk image is malformed' up the lookup pipeline.""" + db_path = _make_db(tmp_path, [_ORANGE_TENDER]) + _truncate(db_path) + monkeypatch.setenv("OPENTENDER_DB_FILE", str(db_path)) + get_settings.cache_clear() + + adapter = OpenTenderAdapter() + assert await adapter.fetch_by_registration("FR", "380129866") == [] + assert not db_path.exists() # corrupt file removed so a re-download can run + + +def test_id_forms_normalisation() -> None: + assert _id_forms("380129866") == ["380129866"] + assert _id_forms("0045274649") == ["0045274649", "45274649"] + assert _id_forms("0056.58.214") == ["0056.58.214", "005658214", "5658214"] + assert _id_forms("") == [] + assert _id_forms(" ") == [] + + # --------------------------------------------------------------------- # _bridge_identifier # ---------------------------------------------------------------------