Preserve configuration fallbacks and refund superseded extraction attempts

Author: nonprofittechy

efile_app/efile/api/case_type_config.py

@@ -11,7 +11,9 @@ def get_case_type_config(request):
         jurisdiction = request.GET.get("jurisdiction") or request.session.get("jurisdiction")
 
         # Use the new jurisdiction-aware configuration loader
-        config_data = config_loader.load_jurisdiction_config(jurisdiction)
+        config_data = (
+            config_loader.base_config if jurisdiction is None else config_loader.load_jurisdiction_config(jurisdiction)
+        )
 
         # Process case types to ensure proper inheritance from base_case_types
         processed_case_types = {}

efile_app/efile/context_processors.py

@@ -2,7 +2,7 @@
 Context processors for jurisdiction-aware templates
 """
 
-from .utils.config_loader import config_loader
+from .utils.config_loader import InvalidJurisdiction, config_loader
 
 
 def jurisdiction_context(request):
@@ -21,7 +21,12 @@ def jurisdiction_context(request):
 
     # Generic pages have no selected state. Do not pass absence through the
     # strict request-to-configuration boundary as a jurisdiction identifier.
-    config = config_loader.load_jurisdiction_config(current_jurisdiction) if current_jurisdiction else {}
+    try:
+        config = config_loader.load_jurisdiction_config(current_jurisdiction) if current_jurisdiction else {}
+    except InvalidJurisdiction:
+        # Error templates also run context processors. Invalid request input
+        # must not prevent Django from rendering the original error response.
+        config = {}
 
     return {
         "jurisdiction": current_jurisdiction,

efile_app/efile/services/document_extractions.py

@@ -11,7 +11,7 @@
 
 from django.conf import settings
 from django.db import connection, transaction
-from django.db.models import Q
+from django.db.models import F, Q
 from django.utils import timezone
 from markitdown import MarkItDown
 from pypdf import PdfReader, PdfWriter
@@ -38,6 +38,10 @@
 logger = logging.getLogger(__name__)
 
 
+class ExtractionSuperseded(Exception):
+    """The requested analysis changed; this is not a processing failure."""
+
+
 def queue_document_extraction(document):
     """Create or reset the one background extraction job for a lead PDF."""
     if document.role != FilingDocument.Role.LEAD:
@@ -326,14 +330,18 @@ def check_outbound_permission():
                     )
                     .exists()
                 ):
-                    raise RuntimeError("Extraction claim superseded")
-
-            analysis = analyze_document(
-                analysis_path,
-                document.draft.jurisdiction,
-                use_ai=not opted_out,
-                before_outbound=check_outbound_permission,
-            )
+                    raise ExtractionSuperseded
+
+            try:
+                analysis = analyze_document(
+                    analysis_path,
+                    document.draft.jurisdiction,
+                    use_ai=not opted_out,
+                    before_outbound=check_outbound_permission,
+                )
+            except ExtractionSuperseded:
+                _requeue_changed_preference(job_id, claim_token, opted_out)
+                return None
 
     # Keep compatibility with extensions that still return the old flat shape.
     if "guesses" in analysis and isinstance(analysis.get("guesses"), dict):
@@ -353,7 +361,10 @@ def check_outbound_permission():
         job = _current_claim(job_id, claim_token).select_for_update().first()
         if job is None:
             return None
-        if draft is None or draft.ai_assistance_opted_out != opted_out:
+        if draft is None:
+            return None
+        if draft.ai_assistance_opted_out != opted_out:
+            _requeue_changed_preference(job_id, claim_token, opted_out)
             return None
         document = job.document
         # A filer can remove or replace the lead while this worker is running.
@@ -455,6 +466,29 @@ def renew_extraction_lease(job_id, claim_token):
     return bool(_current_claim(job_id, claim_token).update(lease_expires_at=timezone.now() + timedelta(minutes=15)))
 
 
+def _requeue_changed_preference(job_id, claim_token, opted_out):
+    """Refund a superseded attempt without resetting earlier real failures."""
+    now = timezone.now()
+    changed_documents = FilingDocument.objects.filter(draft__ai_assistance_opted_out=not opted_out).values("pk")
+    # Keep the claim predicates on the UPDATE itself, rather than inside a
+    # joined-query subselect, so a concurrent new claim remains protected.
+    return (
+        _current_claim(job_id, claim_token)
+        .filter(document_id__in=changed_documents)
+        .update(
+            status=DocumentExtraction.Status.PENDING,
+            attempts=F("attempts") - 1,
+            claim_token=None,
+            lease_expires_at=None,
+            available_at=now,
+            started_at=None,
+            completed_at=None,
+            error="",
+            updated_at=now,
+        )
+    )
+
+
 def record_extraction_failure(job_id, claim_token, error):
     """Retry transient failures, then expose a manual-entry fallback."""
     with transaction.atomic():

efile_app/efile/tests/test_extraction_claims.py

@@ -99,14 +99,16 @@ def download(*args):
     analyze.assert_not_called()
 
 
-def test_preference_change_during_local_conversion_stops_model_request(lead):
+@pytest.mark.parametrize("requeue", [True, False])
+def test_preference_change_during_local_conversion_stops_model_request(lead, requeue):
     queue_document_extraction(lead)
     job = claim_next_extraction()
 
     def convert(*args):
         lead.draft.ai_assistance_opted_out = True
         lead.draft.save()
-        queue_document_extraction(lead)
+        if requeue:
+            queue_document_extraction(lead)
         return "local text", 1
 
     with (
@@ -120,9 +122,53 @@ def convert(*args):
         patch("efile.services.document_extractions.get_default_model", return_value="test-model"),
         patch("efile.services.document_extractions.extract_fields_from_file") as extract,
     ):
-        with pytest.raises(RuntimeError, match="superseded"):
-            process_document_extraction(job.pk, job.claim_token)
+        assert process_document_extraction(job.pk, job.claim_token) is None
     extract.assert_not_called()
+    assert record_extraction_failure(job.pk, job.claim_token, "Worker exited") is None
+    job.refresh_from_db()
+    assert job.status == DocumentExtraction.Status.PENDING
+    assert job.attempts == 0
+    assert job.error == ""
+    assert claim_next_extraction() is not None
+
+
+@pytest.mark.parametrize("attempts_before", [0, 2])
+@pytest.mark.parametrize("initial_opted_out", [True, False])
+@override_settings(DOCUMENT_EXTRACTION_MAX_ATTEMPTS=3)
+def test_preference_change_at_completion_refunds_only_obsolete_attempt(lead, attempts_before, initial_opted_out):
+    FilingDraft.objects.filter(pk=lead.draft_id).update(ai_assistance_opted_out=initial_opted_out)
+    job = queue_document_extraction(lead)
+    DocumentExtraction.objects.filter(pk=job.pk).update(attempts=attempts_before)
+    claimed = claim_next_extraction()
+
+    def analyze(*args, **kwargs):
+        # Simulate an update that does not use the normal requeue endpoint.
+        FilingDraft.objects.filter(pk=lead.draft_id).update(ai_assistance_opted_out=not initial_opted_out)
+        return {"document title": "obsolete result"}
+
+    with (
+        patch(
+            "efile.services.document_extractions.S3UploadHandler",
+            return_value=Mock(download_file=Mock(return_value={"success": True})),
+        ),
+        patch("efile.services.document_extractions.limited_pdf", fake_pdf),
+        patch("efile.services.document_extractions.analyze_document", side_effect=analyze),
+    ):
+        assert process_document_extraction(job.pk, claimed.claim_token) is None
+    # The supervisor must treat the child's normal exit as a no-op.
+    assert record_extraction_failure(job.pk, claimed.claim_token, "Worker exited") is None
+    job.refresh_from_db()
+    assert job.status == DocumentExtraction.Status.PENDING
+    assert job.attempts == attempts_before
+    assert job.claim_token is None
+    assert job.lease_expires_at is None
+    assert job.error == ""
+    lead.draft.refresh_from_db()
+    assert lead.draft.extracted_guesses == {}
+    new_claim = claim_next_extraction()
+    assert new_claim is not None
+    assert new_claim.attempts == attempts_before + 1
+    assert new_claim.claim_token != claimed.claim_token
 
 
 @override_settings(DOCUMENT_EXTRACTION_MAX_ATTEMPTS=1)

efile_app/efile/tests/test_security_boundaries.py

@@ -1,13 +1,15 @@
 """Regression checks for public routes, configuration input, and filing logs."""
 
 import logging
+from copy import deepcopy
 from unittest.mock import Mock, patch
 
 import pytest
 from django.contrib.sessions.backends.signed_cookies import SessionStore
 from django.test import RequestFactory, override_settings
 
 from efile.api.filing_views import get_tyler_token
+from efile.utils.config_loader import config_loader
 from efile.views.session_api import forward_final_filing
 
 
@@ -28,6 +30,45 @@ def test_configuration_endpoints_reject_path_aliases(client, endpoint):
     assert response.status_code == 400
 
 
+@pytest.mark.django_db
+def test_case_type_configuration_without_jurisdiction_uses_cached_base(client):
+    with patch.object(config_loader, "load_jurisdiction_config") as load:
+        response = client.get("/api/case-type-config/")
+    assert response.status_code == 200
+    config = response.json()["config"]
+    assert config["jurisdiction"] is None
+    assert config["case_types"] == config_loader.base_config["base_case_types"]
+    assert config["base_case_types"] == config_loader.base_config["base_case_types"]
+    load.assert_not_called()
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("jurisdiction", ["bogus", "../states/illinois"])
+def test_explicit_unknown_case_type_jurisdiction_still_rejected(client, jurisdiction):
+    assert client.get("/api/case-type-config/", {"jurisdiction": jurisdiction}).status_code == 400
+
+
+@pytest.mark.django_db
+def test_template_with_unknown_query_jurisdiction_renders(client):
+    response = client.get("/about/", {"jurisdiction": "bogus"})
+    assert response.status_code == 200
+    assert response.context["config"] == {}
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("url", ["/missing-page/?jurisdiction=bogus", "/jurisdiction/bogus/missing-page/"])
+def test_unknown_jurisdiction_does_not_break_request_aware_404(client, settings, url):
+    templates = deepcopy(settings.TEMPLATES)
+    templates[0]["APP_DIRS"] = False
+    templates[0]["OPTIONS"]["loaders"] = [
+        ("django.template.loaders.locmem.Loader", {"404.html": "missing-page-marker {{ config|length }}"})
+    ]
+    with override_settings(DEBUG=False, TEMPLATES=templates):
+        response = client.get(url)
+    assert response.status_code == 404
+    assert response.content == b"missing-page-marker 0"
+
+
 @pytest.mark.parametrize("status", [201, 400, 500])
 @override_settings(SUFFOLK_EFILE_API_KEY="secret-api-key-marker")
 def test_submission_logs_exclude_secrets_and_contents(caplog, status):