diff --git a/README.md b/README.md index 06e9b1f..84fd801 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Route Mode sends requests containing sensitive data to a local LLM such as Ollam ## What It Catches -**Personal data**: Names, locations, emails, phone numbers, credit cards, IBANs, IP addresses, and EU VAT numbers. Detection is multilingual. +**Personal data**: Names, locations, emails, phone numbers, credit cards, IBANs, IP addresses, EU VAT numbers, and Chinese Resident ID numbers. Detection is multilingual. **Secrets**: API keys for providers such as OpenAI, Anthropic, Stripe, AWS, and GitHub; SSH and PEM private keys; JWT tokens; bearer tokens; passwords; and connection strings. diff --git a/benchmarks/pii-accuracy/taxonomy.ts b/benchmarks/pii-accuracy/taxonomy.ts index f589327..8a8058e 100644 --- a/benchmarks/pii-accuracy/taxonomy.ts +++ b/benchmarks/pii-accuracy/taxonomy.ts @@ -12,6 +12,7 @@ export const CONFIGURED_PII_ENTITIES = [ "PERSON", "PHONE_NUMBER", "VAT_CODE", + "CN_RESIDENT_ID", ] as const; export const SUPPORTED_LANGUAGES = ["en", "de", "es", "fr", "it", "nl", "pl", "pt", "ro"] as const; diff --git a/benchmarks/pii-accuracy/test-data/core.yaml b/benchmarks/pii-accuracy/test-data/core.yaml index ba003d6..e323737 100644 --- a/benchmarks/pii-accuracy/test-data/core.yaml +++ b/benchmarks/pii-accuracy/test-data/core.yaml @@ -416,3 +416,25 @@ cases: - entity: VAT_CODE text: FR40303265045 match: exact + + - id: core_cn_resident_id + suite: core + category: core + split: test + language: en + text: "The passenger's Chinese ID card number is 360426199101010071." + expected: + - entity: CN_RESIDENT_ID + text: "360426199101010071" + match: exact + + - id: core_cn_resident_id_luhn_valid + suite: core + category: core + split: test + language: en + text: "Resident ID 110105199001010256 was verified at check-in." + expected: + - entity: CN_RESIDENT_ID + text: "110105199001010256" + match: exact diff --git a/benchmarks/pii-accuracy/test-data/precision.yaml b/benchmarks/pii-accuracy/test-data/precision.yaml index 4c5b4f7..71e7fa8 100644 --- a/benchmarks/pii-accuracy/test-data/precision.yaml +++ b/benchmarks/pii-accuracy/test-data/precision.yaml @@ -284,3 +284,12 @@ cases: entities: [VAT_CODE] text: "Die Referenz DE136695977 hat keine gueltige Pruefsumme." expected: [] + + - id: precision_invalid_cn_resident_id_checksum + suite: precision + category: precision + split: test + language: en + entities: [CN_RESIDENT_ID] + text: "Reference 360426199101010072 has an invalid check digit." + expected: [] diff --git a/config.example.yaml b/config.example.yaml index 5425b6f..5847829 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -91,6 +91,7 @@ pii_detection: - IBAN_CODE - IP_ADDRESS - VAT_CODE # EU VAT number (any member state; requires country prefix) + - CN_RESIDENT_ID # Chinese Resident Identity Card number (18 characters) # Secrets Detection settings (Secrets Shield) # Detects private keys, API keys, tokens and other secret credentials in requests diff --git a/detector/detector/app.py b/detector/detector/app.py index 1cd2ca9..410a558 100644 --- a/detector/detector/app.py +++ b/detector/detector/app.py @@ -55,7 +55,7 @@ def health() -> dict[str, str]: @app.post("/analyze", response_model=list[Entity]) def analyze(req: AnalyzeRequest) -> list[Entity]: - deterministic = detect_deterministic(req.text, req.phone_regions) + deterministic = detect_deterministic(req.text, req.phone_regions, req.entities) semantic = detect_semantic(req.text, req.score_threshold) spans = merge(deterministic, semantic, req.entities, 0.0) to_u16 = _utf16_mapper(req.text) diff --git a/detector/detector/deterministic.py b/detector/detector/deterministic.py index 052a632..78175a5 100644 --- a/detector/detector/deterministic.py +++ b/detector/detector/deterministic.py @@ -14,9 +14,11 @@ import phonenumbers from stdnum import iban as _iban_lib from stdnum import luhn as _luhn +from stdnum.cn import ric as _cn_ric from stdnum.eu import vat as _eu_vat from .entities import ( + CN_RESIDENT_ID, CREDIT_CARD, EMAIL_ADDRESS, IBAN_CODE, @@ -55,6 +57,9 @@ r"(?" + _VAT_CC + r")[ ]?(?P[0-9A-Za-z]{8,12})(?![A-Za-z0-9]))", re.IGNORECASE, ) +# Chinese Resident ID: 17 digits + digit/X check character. ASCII-only boundaries +# because CJK characters are `\w` and IDs often sit directly next to them. +_CN_RIC_RE = re.compile(r"(? list[Span]: @@ -115,6 +120,15 @@ def _vat(text: str) -> list[Span]: return out +def _cn_resident_id(text: str) -> list[Span]: + # stdnum checks the checksum, birth date, and birth-place code. + return [ + Span(CN_RESIDENT_ID, m.start(), m.end(), 1.0) + for m in _CN_RIC_RE.finditer(text) + if _cn_ric.is_valid(m.group()) + ] + + def _credit_card(text: str) -> list[Span]: out: list[Span] = [] for m in _CC_RE.finditer(text): @@ -156,7 +170,11 @@ def _phone(text: str, phone_regions: list[str] | None = None) -> list[Span]: return [s for i, s in enumerate(out) if not any(overlaps(s, prev) for prev in out[:i])] -def detect_deterministic(text: str, phone_regions: list[str] | None = None) -> list[Span]: +def detect_deterministic( + text: str, + phone_regions: list[str] | None = None, + entities: list[str] | None = None, +) -> list[Span]: if not text: return [] @@ -166,9 +184,17 @@ def detect_deterministic(text: str, phone_regions: list[str] | None = None) -> l ordered += _ipv4(text) ordered += _iban(text) ordered += _vat(text) + # Before credit cards: about 1 in 10 valid IDs also passes Luhn. + ordered += _cn_resident_id(text) ordered += _credit_card(text) ordered += _phone(text, phone_regions) + # Filter before overlap resolution so a type the caller did not request + # cannot suppress a requested one (e.g. an ID that also passes Luhn). + if entities: + allow = set(entities) + ordered = [s for s in ordered if s.entity_type in allow] + accepted: list[Span] = [] for span in ordered: if any(overlaps(span, a) for a in accepted): diff --git a/detector/detector/entities.py b/detector/detector/entities.py index bc8ee73..2575db3 100644 --- a/detector/detector/entities.py +++ b/detector/detector/entities.py @@ -9,7 +9,8 @@ from dataclasses import dataclass # Entity type strings returned on the /analyze response. The Presidio drop-in -# set plus VAT_CODE (EU VAT numbers, checksum-validated). Other structured types +# set plus VAT_CODE (EU VAT numbers) and CN_RESIDENT_ID (Chinese Resident +# Identity Card numbers), both checksum-validated. Other structured types # (org/address/fiscal/BIC/crypto) are not currently emitted. PERSON = "PERSON" LOCATION = "LOCATION" @@ -19,6 +20,7 @@ IBAN_CODE = "IBAN_CODE" IP_ADDRESS = "IP_ADDRESS" VAT_CODE = "VAT_CODE" +CN_RESIDENT_ID = "CN_RESIDENT_ID" @dataclass(frozen=True) diff --git a/detector/tests/test_analyze.py b/detector/tests/test_analyze.py index 35d99d8..3bf3909 100644 --- a/detector/tests/test_analyze.py +++ b/detector/tests/test_analyze.py @@ -92,6 +92,19 @@ def test_entity_filter(client): assert types == {"IBAN_CODE"} +def test_cn_resident_id_label_follows_requested_entities(client): + # This valid ID also passes Luhn: configs without CN_RESIDENT_ID keep masking it. + text = "身份证110105199001010256。" + for entities, expected in [ + (["CREDIT_CARD", "CN_RESIDENT_ID"], "CN_RESIDENT_ID"), + (["CREDIT_CARD"], "CREDIT_CARD"), + ]: + r = client.post("/analyze", json={"text": text, "entities": entities}) + assert [(e["entity_type"], text[e["start"] : e["end"]]) for e in r.json()] == [ + (expected, "110105199001010256") + ] + + def test_phone_regions_control_national_formats(client): text = "English ticket text with Indian callback 98765 43210." r = client.post( diff --git a/detector/tests/test_deterministic.py b/detector/tests/test_deterministic.py index d2194a7..45a0867 100644 --- a/detector/tests/test_deterministic.py +++ b/detector/tests/test_deterministic.py @@ -4,6 +4,7 @@ from detector.deterministic import detect_deterministic from detector.entities import ( + CN_RESIDENT_ID, CREDIT_CARD, EMAIL_ADDRESS, IBAN_CODE, @@ -108,6 +109,45 @@ def test_vat_word_prefix_does_not_swallow_following_vat(): assert (VAT_CODE, "DE136695976") in types_texts(text) +# --- Chinese Resident ID (stdnum-validated) --- +def test_cn_resident_id_next_to_cjk_text(): + text = "张三的身份证号是360426199101010071请核对。" + assert types_texts(text) == [(CN_RESIDENT_ID, "360426199101010071")] + + +def test_cn_resident_id_x_check_character(): + assert (CN_RESIDENT_ID, "11010519900101007X") in types_texts("ID 11010519900101007X ok") + assert (CN_RESIDENT_ID, "11010519900101007x") in types_texts("id 11010519900101007x ok") + + +def test_cn_resident_id_wins_over_luhn_valid_credit_card(): + # This valid ID also passes Luhn; it must not be labeled CREDIT_CARD. + assert types_texts("身份证110105199001010256。") == [(CN_RESIDENT_ID, "110105199001010256")] + + +def test_cn_resident_id_invalid_rejected(): + for number in [ + "360426199101010072", # wrong check digit + "360426199102300070", # Feb 30 birth date, correct check digit + "999999199101010072", # unknown region code, correct check digit + ]: + assert all(t != CN_RESIDENT_ID for t, _ in types_texts(f"身份证号{number}。")) + + +def test_cn_resident_id_not_matched_inside_longer_token(): + for text in ["1360426199101010071", "3604261991010100710", "A360426199101010071"]: + assert all(t != CN_RESIDENT_ID for t, _ in types_texts(text)) + + +def test_unrequested_type_does_not_suppress_requested_one(): + # Without CN_RESIDENT_ID requested, a Luhn-valid ID stays masked as CREDIT_CARD. + text = "身份证110105199001010256。" + assert [ + (s.entity_type, text[s.start : s.end]) + for s in detect_deterministic(text, entities=[CREDIT_CARD]) + ] == [(CREDIT_CARD, "110105199001010256")] + + # --- Email / IP --- def test_email(): assert (EMAIL_ADDRESS, "john.doe@company.com") in types_texts("at john.doe@company.com") diff --git a/docs/concepts/pii-detection.mdx b/docs/concepts/pii-detection.mdx index b662ed2..ee9c1a0 100644 --- a/docs/concepts/pii-detection.mdx +++ b/docs/concepts/pii-detection.mdx @@ -33,8 +33,9 @@ Backend selection does not change the deterministic layer or response contract. | `CREDIT_CARD` | 4111 1111 1111 1111 | | `IP_ADDRESS` | 192.168.1.1 | | `VAT_CODE` | EU VAT number, e.g. `DE136695976`, `IT00743110157`, `FR40303265045` | +| `CN_RESIDENT_ID` | Chinese Resident Identity Card number, e.g. `360426199101010071`, `11010519900101007X` | -Names and locations use the multilingual model. IBAN, credit card, email, and IP work internationally. Phone numbers are `+` international-only by default; set `phone_regions` for local formats. `VAT_CODE` requires a country prefix and is validated with `python-stdnum`. +Names and locations use the multilingual model. IBAN, credit card, email, and IP work internationally. Phone numbers are `+` international-only by default; set `phone_regions` for local formats. `VAT_CODE` requires a country prefix and `CN_RESIDENT_ID` requires all 18 characters; both are validated with `python-stdnum`. ## Languages and phone regions diff --git a/docs/configuration/pii-detection.mdx b/docs/configuration/pii-detection.mdx index 80f05cc..bd0a317 100644 --- a/docs/configuration/pii-detection.mdx +++ b/docs/configuration/pii-detection.mdx @@ -18,6 +18,7 @@ pii_detection: - IBAN_CODE - IP_ADDRESS - VAT_CODE + - CN_RESIDENT_ID ``` ## Options @@ -66,8 +67,9 @@ Keep the list focused. More regions can mean more false positives on IDs and tic | `IBAN_CODE` | DE89 3704 0044 0532 0130 00 | | `IP_ADDRESS` | 192.168.1.1 | | `VAT_CODE` | EU VAT number, e.g. `DE136695976`, `IT00743110157`, `FR40303265045` | +| `CN_RESIDENT_ID` | Chinese Resident Identity Card number, e.g. `360426199101010071`, `11010519900101007X` | -Names and locations are multilingual. IBAN, credit card, email, IP, and `+` phone numbers work internationally. `VAT_CODE` requires a country prefix and is validated with `python-stdnum`. +Names and locations are multilingual. IBAN, credit card, email, IP, and `+` phone numbers work internationally. `VAT_CODE` requires a country prefix and `CN_RESIDENT_ID` requires all 18 characters; both are validated with `python-stdnum`. ## Score Threshold diff --git a/docs/configuration/semantic-backends.mdx b/docs/configuration/semantic-backends.mdx index 19b7b72..3fd38d6 100644 --- a/docs/configuration/semantic-backends.mdx +++ b/docs/configuration/semantic-backends.mdx @@ -6,7 +6,8 @@ description: Select and verify the semantic detector backend PasteGuard's detector combines two independent layers: - The **deterministic layer** validates structured values such as email addresses, - phone numbers, credit cards, IBANs, IP addresses, and VAT numbers. + phone numbers, credit cards, IBANs, IP addresses, VAT numbers, and Chinese + Resident ID numbers. - The **semantic backend** detects names, locations, and street addresses. Changing the backend does not affect deterministic detection. diff --git a/src/config.ts b/src/config.ts index 0357e23..98f4887 100644 --- a/src/config.ts +++ b/src/config.ts @@ -144,6 +144,7 @@ const PIIDetectionSchema = z.object({ "IBAN_CODE", "IP_ADDRESS", "VAT_CODE", + "CN_RESIDENT_ID", ]), scan_roles: scanRolesField, });