From c6876a5c9f549114ba95dcc507b8cbf01aa0b65d Mon Sep 17 00:00:00 2001 From: Alan Smith Date: Sun, 12 Jul 2026 01:55:21 +0200 Subject: [PATCH 1/6] Harden Track 1 runtime and release pipeline --- .env.example | 7 +- .gitattributes | 9 + .github/workflows/container-release.yml | 373 +++++++++ .gitignore | 8 + Dockerfile | 12 +- README.md | 455 +++++++---- docs/amd-pod-environment.md | 29 +- docs/developer-handoff-2026-07-11.md | 11 +- docs/final-runtime-architecture.md | 132 ++++ docs/track-1-official-contract.md | 135 ++++ eval/benchmark_local_routes.py | 437 ++++++++--- eval/common.py | 142 +++- eval/compare_routes.py | 510 +++++++++++- eval/constraints.py | 117 +++ eval/datasets/README.md | 40 +- eval/datasets/calibration-v1.json | 363 +++++++++ eval/datasets/holdout-v1.json | 369 +++++++++ eval/evaluate_policy.py | 209 +++++ eval/report.py | 19 +- eval/run_routes.py | 332 ++++++-- policy/README.md | 33 + policy/evidence.schema.json | 75 ++ pyproject.toml | 3 + scripts/build_container.sh | 57 +- scripts/probe_environment.py | 5 +- scripts/run_container_constrained.sh | 84 ++ ...e_test_vllm.py => smoke_test_llama_cpp.py} | 162 ++-- src/fallthrough/config.py | 104 ++- src/fallthrough/fireworks/client.py | 43 +- src/fallthrough/inference.py | 169 ++-- src/fallthrough/local/__init__.py | 16 +- src/fallthrough/local/backend.py | 31 +- src/fallthrough/local/gguf_metadata.py | 398 ++++++++++ src/fallthrough/local/identity.py | 124 +-- src/fallthrough/local/llama_cpp_backend.py | 742 ++++++++++++++++++ src/fallthrough/local/vllm_backend.py | 389 --------- src/fallthrough/main.py | 70 +- src/fallthrough/policy/evidence.py | 216 +++++ src/fallthrough/policy/route_table.py | 67 +- src/fallthrough/routes/deterministic.py | 5 + src/fallthrough/routes/fireworks_gemma.py | 4 + src/fallthrough/routes/fireworks_model.py | 43 +- src/fallthrough/routes/local_model.py | 12 +- src/fallthrough/runtime/adapter.py | 17 +- src/fallthrough/runtime/engine.py | 65 +- src/fallthrough/tasks/classifier.py | 36 +- src/fallthrough/tasks/features.py | 24 +- src/fallthrough/tasks/labels.py | 102 ++- src/fallthrough/tasks/requirements.py | 160 ++++ src/fallthrough/telemetry/observations.py | 14 + src/fallthrough/validation/__init__.py | 9 +- src/fallthrough/validation/code.py | 243 +++++- src/fallthrough/validation/entities.py | 82 +- src/fallthrough/validation/registry.py | 68 +- src/fallthrough/validation/sentiment.py | 145 +++- src/fallthrough/validation/summary.py | 134 ++++ tests/fixtures/mock_fireworks_server.py | 97 +++ tests/test_adapter.py | 40 + tests/test_classifier.py | 80 ++ tests/test_config.py | 162 +++- tests/test_deterministic_route.py | 29 +- tests/test_environment_scripts.py | 151 ++-- tests/test_eval_tools.py | 499 +++++++++++- tests/test_fireworks.py | 154 +++- tests/test_gguf_metadata.py | 188 +++++ tests/test_local_backend.py | 478 ++++++----- tests/test_local_benchmark.py | 204 ++++- tests/test_local_identity.py | 78 +- tests/test_main.py | 297 ++++++- tests/test_policy.py | 155 ++++ tests/test_route_retries.py | 49 +- tests/test_runtime.py | 57 ++ tests/test_validation.py | 397 ++++++++++ 73 files changed, 9341 insertions(+), 1434 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/container-release.yml create mode 100644 docs/final-runtime-architecture.md create mode 100644 docs/track-1-official-contract.md create mode 100644 eval/constraints.py create mode 100644 eval/datasets/calibration-v1.json create mode 100644 eval/datasets/holdout-v1.json create mode 100644 eval/evaluate_policy.py create mode 100644 policy/README.md create mode 100644 policy/evidence.schema.json create mode 100644 scripts/run_container_constrained.sh rename scripts/{smoke_test_vllm.py => smoke_test_llama_cpp.py} (58%) create mode 100644 src/fallthrough/local/gguf_metadata.py create mode 100644 src/fallthrough/local/llama_cpp_backend.py delete mode 100644 src/fallthrough/local/vllm_backend.py create mode 100644 src/fallthrough/policy/evidence.py create mode 100644 src/fallthrough/tasks/requirements.py create mode 100644 src/fallthrough/validation/summary.py create mode 100644 tests/fixtures/mock_fireworks_server.py create mode 100644 tests/test_gguf_metadata.py diff --git a/.env.example b/.env.example index 489af3f..eff1d29 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,7 @@ FIREWORKS_API_KEY= -# Evaluator/hackathon-provided endpoint ONLY. Never the public Fireworks API -# URL (compliance addendum), and there is deliberately no default. +# Evaluator/hackathon-provided endpoint; there is no public fallback. FIREWORKS_BASE_URL= # Enforced strictly when set; an empty value means deny-all. @@ -15,3 +14,7 @@ ALLOWED_MODELS= # FALLTHROUGH_GEMMA_MODEL= # FALLTHROUGH_POLICY=hybrid-baseline # FALLTHROUGH_FORMAT_RETRIES=1 +# FALLTHROUGH_LOCAL_BACKEND=none +# FALLTHROUGH_LOCAL_MODEL=/models/fallthrough.gguf +# FALLTHROUGH_LOCAL_THREADS=2 +# FALLTHROUGH_LOCAL_CONTEXT_SIZE=1024 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e139d4d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +* text=auto +*.py text eol=lf +*.sh text eol=lf +*.toml text eol=lf +*.md text eol=lf +*.json text eol=lf +*.jsonl text eol=lf +.env.example text eol=lf +.gitignore text eol=lf diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml new file mode 100644 index 0000000..8542819 --- /dev/null +++ b/.github/workflows/container-release.yml @@ -0,0 +1,373 @@ +name: Container CI and release + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + inputs: + push_image: + description: Push the verified image to public GHCR using an immutable commit-SHA tag + required: true + default: false + type: boolean + +concurrency: + group: container-release-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + quality: + name: Python 3.10 tests and static checks + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: pip + cache-dependency-path: | + pyproject.toml + requirements.lock + + - name: Install pinned runtime and development dependencies + run: | + python -m pip install --requirement requirements.lock + python -m pip install --no-build-isolation --editable ".[dev]" + + - name: Run test suite + run: python -m pytest -q + + - name: Run Ruff lint + run: python -m ruff check . + + - name: Check Ruff formatting + run: python -m ruff format --check . + + - name: Run strict type checks + run: python -m mypy src/fallthrough eval scripts + + container: + name: Build and constrain linux/amd64 image + needs: quality + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + packages: write + env: + CI_IMAGE: fallthrough-ci:${{ github.sha }} + MAX_COMPRESSED_BYTES: "10000000000" + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Build clean linux/amd64 image + run: | + set -euo pipefail + docker build \ + --no-cache \ + --pull \ + --platform linux/amd64 \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --tag "${CI_IMAGE}" \ + . + + architecture="$(docker image inspect --format '{{.Architecture}}' "${CI_IMAGE}")" + operating_system="$(docker image inspect --format '{{.Os}}' "${CI_IMAGE}")" + test "${operating_system}" = "linux" + test "${architecture}" = "amd64" + + - name: Run exact-I/O cold-start smoke under judging constraints + id: smoke + run: | + set -euo pipefail + smoke_root="${RUNNER_TEMP}/fallthrough-smoke" + container_name="fallthrough-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + rm -rf "${smoke_root}" + mkdir -p "${smoke_root}/input" "${smoke_root}/output" + cleanup() { + docker rm --force "${container_name}" >/dev/null 2>&1 || true + rm -rf "${smoke_root}" + } + trap cleanup EXIT + printf '%s' '[{"task_id":"startup","prompt":"Calculate 1 + 1."}]' \ + > "${smoke_root}/input/tasks.json" + + started_ns="$(date +%s%N)" + if ! timeout --signal=TERM --kill-after=5s 59s \ + docker run --rm \ + --name "${container_name}" \ + --platform linux/amd64 \ + --memory=4000000000 \ + --memory-swap=4000000000 \ + --cpus=2 \ + --pids-limit=256 \ + --read-only \ + --network=none \ + --cap-drop=ALL \ + --security-opt=no-new-privileges \ + --tmpfs /tmp:rw,noexec,nosuid,size=64m \ + --env FALLTHROUGH_POLICY=deterministic-only \ + --volume "${smoke_root}/input:/input:ro" \ + --volume "${smoke_root}/output:/output" \ + "${CI_IMAGE}" \ + --input /input/tasks.json \ + --output /output/results.json + then + echo "::error::The constrained deterministic smoke failed or exceeded 59 seconds." + exit 1 + fi + ended_ns="$(date +%s%N)" + elapsed_ms="$(( (ended_ns - started_ns) / 1000000 ))" + + python3 - "${smoke_root}/output/results.json" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as handle: + actual = json.load(handle) + + expected = [{"task_id": "startup", "answer": "2"}] + if actual != expected: + raise SystemExit(f"unexpected exact Track 1 output: {actual!r}") + PY + + if (( elapsed_ms >= 60000 )); then + echo "::error::Cold-start smoke took ${elapsed_ms} ms; the limit is below 60000 ms." + exit 1 + fi + echo "elapsed_ms=${elapsed_ms}" >> "${GITHUB_OUTPUT}" + printf '### Constrained container smoke\n\n- Cold start plus deterministic task: `%s ms`\n' \ + "${elapsed_ms}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Run eight-family hybrid batch against isolated mock Fireworks + run: | + set -euo pipefail + smoke_root="${RUNNER_TEMP}/fallthrough-hybrid-smoke" + network_name="fallthrough-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mock_name="fallthrough-mock-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + app_name="fallthrough-hybrid-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + rm -rf "${smoke_root}" + mkdir -p "${smoke_root}/input" "${smoke_root}/output" + cleanup() { + docker rm --force "${app_name}" "${mock_name}" >/dev/null 2>&1 || true + docker network rm "${network_name}" >/dev/null 2>&1 || true + rm -rf "${smoke_root}" + } + trap cleanup EXIT + + python3 - "${smoke_root}/input/tasks.json" <<'PY' + import json + import sys + + tasks = [ + {"task_id": "arithmetic", "prompt": "Calculate 19 * 7. Return only the number."}, + {"task_id": "factual", "prompt": "What is the capital of France? Return the city name only."}, + {"task_id": "sentiment", "prompt": "Classify the sentiment as Positive or Negative. Return one label only.\nReview: Everything works perfectly."}, + {"task_id": "summary", "prompt": "Summarize in exactly one sentence: The library extended its weekend hours after community requests."}, + {"task_id": "ner", "prompt": "Extract all named entities as JSON objects with text and type. Text: Alice visited Berlin."}, + {"task_id": "debug", "prompt": "Fix this Python and return corrected code only:\ndef add(a, b)\n return a + b"}, + {"task_id": "logic", "prompt": "Every glorp is blue. Tavi is a glorp. What follows? Return the conclusion only."}, + {"task_id": "codegen", "prompt": "Write a Python function named greet that returns the string hello. Return code only."}, + {"task_id": "isolated-failure", "prompt": "What is MOCK_FAILURE? Return only the answer."}, + ] + with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump(tasks, handle) + PY + + docker network create "${network_name}" >/dev/null + docker run --detach --rm \ + --name "${mock_name}" \ + --network "${network_name}" \ + --read-only \ + --cap-drop=ALL \ + --security-opt=no-new-privileges \ + --volume "${GITHUB_WORKSPACE}/tests/fixtures/mock_fireworks_server.py:/fixture.py:ro" \ + --entrypoint python \ + "${CI_IMAGE}" /fixture.py >/dev/null + + for _ in $(seq 1 30); do + if docker logs "${mock_name}" 2>&1 | grep -q mock-fireworks-ready; then + break + fi + sleep 0.2 + done + docker logs "${mock_name}" 2>&1 | grep -q mock-fireworks-ready + + timeout --signal=TERM --kill-after=5s 599s \ + docker run --rm \ + --name "${app_name}" \ + --network "${network_name}" \ + --platform linux/amd64 \ + --memory=4000000000 \ + --memory-swap=4000000000 \ + --cpus=2 \ + --pids-limit=256 \ + --read-only \ + --cap-drop=ALL \ + --security-opt=no-new-privileges \ + --tmpfs /tmp:rw,noexec,nosuid,size=64m \ + --env FIREWORKS_API_KEY=ci-fixture-not-a-secret \ + --env FIREWORKS_BASE_URL=http://${mock_name}:8080/v1 \ + --env ALLOWED_MODELS=ci-fixture-gemma \ + --env FALLTHROUGH_TRANSPORT_RETRIES=0 \ + --env FALLTHROUGH_FORMAT_RETRIES=0 \ + --env FALLTHROUGH_MAX_CONCURRENCY=4 \ + --volume "${smoke_root}/input:/input:ro" \ + --volume "${smoke_root}/output:/output" \ + "${CI_IMAGE}" + + python3 - "${smoke_root}/output/results.json" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as handle: + rows = json.load(handle) + expected = [ + {"task_id": "arithmetic", "answer": "133"}, + {"task_id": "factual", "answer": "Paris"}, + {"task_id": "sentiment", "answer": "Positive"}, + {"task_id": "summary", "answer": "The library extended weekend hours after community requests."}, + {"task_id": "ner", "answer": '[{"text":"Alice","type":"PERSON"},{"text":"Berlin","type":"LOCATION"}]'}, + {"task_id": "debug", "answer": "def add(a, b):\n return a + b"}, + {"task_id": "logic", "answer": "Tavi is blue."}, + {"task_id": "codegen", "answer": 'def greet():\n return "hello"'}, + {"task_id": "isolated-failure", "answer": ""}, + ] + if rows != expected: + raise SystemExit(f"unexpected hybrid batch output: {rows!r}") + PY + + printf -- '- Hybrid mock-proxy batch: `8 families + isolated failure verified`\n' \ + >> "${GITHUB_STEP_SUMMARY}" + + - name: Check compressed docker-save size proxy + id: size + run: | + set -euo pipefail + compressed_bytes="$(docker save "${CI_IMAGE}" | gzip -c | wc -c | tr -d '[:space:]')" + if (( compressed_bytes >= MAX_COMPRESSED_BYTES )); then + echo "::error::Compressed docker-save proxy is ${compressed_bytes} bytes; it must be below ${MAX_COMPRESSED_BYTES}." + exit 1 + fi + echo "compressed_bytes=${compressed_bytes}" >> "${GITHUB_OUTPUT}" + printf -- '- Compressed docker-save proxy: `%s bytes` (limit: `< %s`)\n' \ + "${compressed_bytes}" "${MAX_COMPRESSED_BYTES}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Prepare immutable GHCR reference + if: ${{ github.event_name == 'workflow_dispatch' && inputs.push_image }} + id: publish_metadata + run: | + set -euo pipefail + if [[ "${{ github.event.repository.visibility }}" != "public" ]]; then + echo "::error::Publishing is allowed only from a public GitHub repository." + exit 1 + fi + repository_lower="${GITHUB_REPOSITORY,,}" + image_ref="ghcr.io/${repository_lower}:sha-${GITHUB_SHA}" + echo "image_ref=${image_ref}" >> "${GITHUB_OUTPUT}" + + - name: Authenticate to GHCR + if: ${{ github.event_name == 'workflow_dispatch' && inputs.push_image }} + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + printf '%s' "${GHCR_TOKEN}" | \ + docker login ghcr.io --username "${GITHUB_ACTOR}" --password-stdin + + - name: Publish commit-SHA image without overwriting + if: ${{ github.event_name == 'workflow_dispatch' && inputs.push_image }} + env: + IMAGE_REF: ${{ steps.publish_metadata.outputs.image_ref }} + run: | + set -euo pipefail + if docker manifest inspect "${IMAGE_REF}" >/dev/null 2>&1; then + docker pull --platform linux/amd64 "${IMAGE_REF}" + existing_revision="$(docker image inspect \ + --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' \ + "${IMAGE_REF}")" + if [[ "${existing_revision}" != "${GITHUB_SHA}" ]]; then + echo "::error::Existing immutable tag has revision ${existing_revision}, not ${GITHUB_SHA}." + exit 1 + fi + echo "The immutable tag already exists for this commit; it was not overwritten." + else + docker tag "${CI_IMAGE}" "${IMAGE_REF}" + docker push "${IMAGE_REF}" + fi + + - name: Verify anonymous public manifest and linux/amd64 platform + if: ${{ github.event_name == 'workflow_dispatch' && inputs.push_image }} + env: + IMAGE_REF: ${{ steps.publish_metadata.outputs.image_ref }} + run: | + set -euo pipefail + docker logout ghcr.io >/dev/null 2>&1 || true + manifest_path="${RUNNER_TEMP}/fallthrough-manifest.json" + if ! docker manifest inspect --verbose "${IMAGE_REF}" > "${manifest_path}"; then + echo "::error::The GHCR manifest is not anonymously readable. Make the package public, then rerun this workflow." + exit 1 + fi + + python3 - "${manifest_path}" "${MAX_COMPRESSED_BYTES}" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as handle: + manifest = json.load(handle) + + platforms: set[tuple[str, str]] = set() + compressed_layers: dict[str, int] = {} + + def visit(value: object) -> None: + if isinstance(value, dict): + platform = value.get("platform") + if isinstance(platform, dict): + operating_system = platform.get("os") + architecture = platform.get("architecture") + if isinstance(operating_system, str) and isinstance(architecture, str): + platforms.add((operating_system, architecture)) + layers = value.get("layers") + if isinstance(layers, list): + for layer in layers: + if not isinstance(layer, dict): + continue + digest = layer.get("digest") + size = layer.get("size") + if isinstance(digest, str) and isinstance(size, int): + compressed_layers[digest] = size + for child in value.values(): + visit(child) + elif isinstance(value, list): + for child in value: + visit(child) + + visit(manifest) + if ("linux", "amd64") not in platforms: + raise SystemExit(f"published manifest lacks linux/amd64: {sorted(platforms)!r}") + registry_bytes = sum(compressed_layers.values()) + maximum_bytes = int(sys.argv[2]) + if not compressed_layers: + raise SystemExit("published manifest did not expose compressed layer sizes") + if registry_bytes >= maximum_bytes: + raise SystemExit( + f"published compressed layers total {registry_bytes} bytes; " + f"limit is below {maximum_bytes}" + ) + print("verified public manifest platforms:", sorted(platforms)) + print("published compressed layer bytes:", registry_bytes) + PY + + printf -- '- Published image: `%s`\n- Anonymous manifest: verified `linux/amd64`\n' \ + "${IMAGE_REF}" >> "${GITHUB_STEP_SUMMARY}" diff --git a/.gitignore b/.gitignore index a4bb38a..0165eae 100644 --- a/.gitignore +++ b/.gitignore @@ -149,7 +149,15 @@ activemq-data/ # Environments .env +.env.* +!.env.example .envrc +.netrc +.ssh/ +*.key +*.pem +credentials* +secrets* .venv env/ venv/ diff --git a/Dockerfile b/Dockerfile index eaadc46..51b46bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,22 @@ -FROM python:3.12.4-slim-bookworm +FROM python:3.10.20-slim-bookworm + +ARG FALLTHROUGH_PACKAGED_ROUTE_TABLE +ARG FALLTHROUGH_PACKAGED_POLICY_EVIDENCE ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PIP_NO_CACHE_DIR=1 + PIP_NO_CACHE_DIR=1 \ + FALLTHROUGH_PACKAGED_ROUTE_TABLE=${FALLTHROUGH_PACKAGED_ROUTE_TABLE} \ + FALLTHROUGH_PACKAGED_POLICY_EVIDENCE=${FALLTHROUGH_PACKAGED_POLICY_EVIDENCE} WORKDIR /app COPY pyproject.toml requirements.lock README.md LICENSE ./ COPY src ./src +# Schema/README are always present. Measured artifacts are added only after +# compliant evidence exists and are activated through the two non-secret build args. +COPY policy ./policy RUN python -m pip install --no-cache-dir --no-deps --requirement requirements.lock \ && python -m pip install --no-cache-dir --no-build-isolation --no-deps . diff --git a/README.md b/README.md index d686be4..029edf9 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,27 @@ Fallthrough is a verification-first hybrid inference runtime for the AMD Developer Hackathon: ACT II, Track 1. It is designed to minimize scored Fireworks tokens without treating zero remote calls as an architectural goal. Objective arithmetic -is solved and validated locally; semantic tasks use Gemma through the -evaluator-provided Fireworks endpoint unless a measured route table selects a -different evaluator-allowlisted Fireworks model. - -This repository currently implements the handoff's remote-Gemma baseline and -Version 1 hybrid baseline. It intentionally does not include a speculative local -model, a learned router, or the batch risk allocator. Those stages require route -measurements first. +is solved and validated locally; semantic tasks prefer Gemma through the +evaluator-provided Fireworks endpoint when Gemma is allowlisted. If it is not, +the unmeasured baseline uses an actually allowlisted Fireworks model rather than +requiring an unavailable one; a measured route table supersedes either choice. +An entirely deterministic/local batch with zero Fireworks calls is explicitly +valid and uses zero scored Fireworks tokens; it is not required if measurements +show that remote inference is needed to clear the undisclosed accuracy gate. + +This repository implements the deterministic/Fireworks baseline and one optional +local GGUF integration through llama.cpp. The local backend is disabled and +shadow-only until the exact artifact clears constrained held-out gates; no local +model is currently baked into the scored image or selected by the baseline. The evidence and accept/reject record for the teammate implementation is in [docs/merge-review-51a609b.md](docs/merge-review-51a609b.md). The probed development hardware and pod workflow are recorded in [docs/amd-pod-environment.md](docs/amd-pod-environment.md). +The confirmed 4 GB/2-vCPU grader design and rule precedence are recorded in +[docs/final-runtime-architecture.md](docs/final-runtime-architecture.md). +The exact official evaluator/scoring contract and resolved FAQ conflicts are in +[docs/track-1-official-contract.md](docs/track-1-official-contract.md). ## Runtime flow @@ -27,29 +35,30 @@ evaluator JSON arithmetic -> safe AST solver -> exact validation structured -> recognised explicit list operations logic -> uniquely determined comparison puzzles - other kinds -> measured Fireworks route (Gemma baseline) + other kinds -> measured Fireworks route (Gemma preferred when allowed) -> per-task failure isolation and usage trace -> atomic evaluator JSON output ``` The classifier never calls an LLM. The runtime supports single tasks and batches, -preserves batch order, applies a global monotonic deadline with output-time -reserve, bounds each route by a wall-clock timeout, and returns a schema-valid -fallback answer when a Fireworks route fails. +preserves batch order, applies global and strict sub-30-second per-task monotonic +deadlines with output-time reserve, bounds each route by that shared task budget, +and returns a schema-valid fallback answer when a route fails. All eight required areas are represented: factual knowledge, mathematical reasoning, sentiment, summarisation, named entity recognition, code debugging, logic, and code generation. The baseline uses specialized deterministic handling -only where a solver can prove a result and declines otherwise; Gemma performs -meaningful answer generation for the remaining families. +only where a solver can prove a result and declines otherwise. When Gemma is in +the evaluator allowlist, it performs meaningful answer generation for the +remaining families. ## Implemented routes | Route | Scored Fireworks tokens | Current use | |---|---:|---| | `deterministic` | 0 | Provable arithmetic, explicit list operations, uniquely determined comparison puzzles | -| `fw_gemma` | API-reported usage | Primary semantic route and meaningful Gemma work | -| `fw_model:` | API-reported usage | Measured runtime/eval route for a non-selected-Gemma allowlisted model | +| `fw_gemma` | API-reported usage | Late-bound, unmeasured baseline/Gemma-award convenience route | +| `fw_model:` | API-reported usage | Exact evidence-backed route for every allowlisted model, including Gemma | | `fw_sc` | API-reported usage (~k×) | Benchmark-only Gemma self-consistency vote; never registered in the scored runtime | The arithmetic solver uses an AST whitelist and exact `Fraction`/`Decimal` @@ -69,22 +78,29 @@ prose lists whose output format would be a guess. The logic solver resolves when the relations share one dimension, contain no contradiction, and determine the answer uniquely. -Gemma receives compact category-specific instructions. For example, factual QA -requests an answer of at most eight words, sentiment requests one allowed label, -and summarisation preserves the task's own length constraint. There is no large -universal system prompt and no routine self-critique call. - -Remote answers are canonicalized before validation and output: presentation -fences, "The answer is …" wrappers, and surrounding quotes are stripped; the -final number is extracted for arithmetic; sentiment output is mapped onto the -task's own requested labels. Canonicalization is purely local text cleanup and -never invents content. +The evaluator prompt is preserved by default, which avoids needless scored +prompt tokens and cannot override a requested explanation or output shape. A +bounded metadata-only label suffix is added only when trusted sentiment labels +are not already present; an explicit non-English output request receives a +conditional English safeguard required by Track 1. Development measurements +support one model-specific exception: Kimi receives a short, generic +conciseness/completeness reminder only for explanatory factual questions, +non-deterministic math, and long summaries. The rule uses task shape, never +prompt identity. There is no universal system prompt or routine self-critique +call. + +Remote canonicalization is deliberately conservative for the official LLM +judge. It removes unrequested whole-answer code fences, normalizes only +objectively recomputable single-value arithmetic and label-only sentiment, and +preserves factual explanations, multi-part math, logic reasoning, summarisation, +NER output, and requested sentiment reasons. Removing prose after generation +would not save scored tokens and could only reduce accuracy. Remote candidates are checked with conservative category validators where an -objective check exists: requested sentiment labels, NER/JSON shape, explicit -arithmetic, Python syntax, truncation status, and non-empty structural output. -These checks are recorded as evidence and are not described as proof of factual -or summary correctness. When a cheap format validator objectively rejects an +objective check exists: requested sentiment labels/reasons, explicit summary +sentence/bullet/word limits, NER/JSON shape, explicit arithmetic, Python syntax, +truncation status, and non-empty structural output. These checks are recorded as +evidence and are not described as proof of factual or summary correctness. When a cheap format validator objectively rejects an answer (or the completion is empty), one corrective retry is sent (`FALLTHROUGH_FORMAT_RETRIES`, default `1`, `0` disables); every attempt's tokens are accounted, and an answer that still fails validation is rejected @@ -92,70 +108,45 @@ rather than committed. ## Evaluator interface -The official evaluator schema was not present in the repository or developer -handoff. That unresolved requirement is isolated in -`fallthrough.runtime.adapter.EvaluationAdapter`; no evaluator field assumptions -appear in the engine, routes, or policy. - -The adapter currently accepts: - -- an array of task objects; -- `{ "tasks": [...] }`; -- one task object; -- an explicit container and field mapping through environment variables. - -Common task fields are `id`/`task_id` and `prompt`/`question`/`input`. Extra fields -are preserved as task metadata. Output mirrors the input shape by default and -contains exactly one result for each input ID, in input order. Missing, duplicate, -or unexpected result IDs are rejected before writing. - -Example input: +The official Track 1 schema is a top-level list of `task_id`/`prompt` objects. +The output is a top-level list of `task_id`/`answer` objects: ```json -{ - "tasks": [ - {"id": "m1", "prompt": "Calculate 19 * 7."}, - {"id": "q1", "prompt": "Who wrote Hamlet?"} - ] -} +[ + {"task_id": "m1", "prompt": "Calculate 19 * 7."}, + {"task_id": "q1", "prompt": "Who wrote Hamlet?"} +] ``` -Example output shape: - ```json -{ - "results": [ - {"task_id": "m1", "answer": "133"}, - {"task_id": "q1", "answer": "William Shakespeare"} - ] -} +[ + {"task_id": "m1", "answer": "133"}, + {"task_id": "q1", "answer": "William Shakespeare"} +] ``` The second answer above only illustrates the schema; it is not a cached runtime -answer. - -### Official integration details still unresolved - -The repository does not yet contain the organizer's participant guide or runtime -contract. Before submission, confirm: +answer. The adapter preserves input order, writes exactly one row per ID, and +rejects missing, duplicate, or unexpected result IDs. Its envelope/single-task +and alternate-field modes remain development compatibility features isolated at +the boundary; the scored contract uses the lists above. -- the exact evaluator invocation and whether it uses stdin/stdout or fixed paths; -- the official input and output field/container names; -- whether the Fireworks variables or allowed-model encoding use different names; -- the required base image, total deadline, and concurrency limits. +### Final grader contract -Those assumptions are confined to `Settings`, `EvaluationAdapter`, CLI arguments, -and the first Dockerfile line. Confirming them should not require route, solver, or -engine changes. +The submitted `linux/amd64` container receives 4 GB RAM and 2 vCPU, must start in +under 60 seconds, finish each task in under 30 seconds and the batch in under ten +minutes, and remain below 10 GB compressed. It reads `/input/tasks.json`, writes +`/output/results.json`, and exits zero only after a successful batch. Answers must +be in English. The image must be publicly pullable and expose a `linux/amd64` +registry manifest. -When the official schema is published, set these variables or change only the -adapter: +Adapter overrides are development/forward-compatibility controls: | Variable | Purpose | Default | |---|---|---| | `FALLTHROUGH_INPUT_CONTAINER_KEY` | Array key in an input envelope | Auto-detect `tasks` | -| `FALLTHROUGH_ID_FIELD` | Input task identifier field | Auto-detect | -| `FALLTHROUGH_PROMPT_FIELD` | Input prompt field | Auto-detect | +| `FALLTHROUGH_ID_FIELD` | Input task identifier field | Prefer official `task_id` | +| `FALLTHROUGH_PROMPT_FIELD` | Input prompt field | Prefer official `prompt` | | `FALLTHROUGH_OUTPUT_MODE` | `auto`, `list`, `envelope`, or `single` | `auto` | | `FALLTHROUGH_OUTPUT_CONTAINER_KEY` | Output envelope key | `results` | | `FALLTHROUGH_OUTPUT_ID_FIELD` | Output result identifier field | `task_id` | @@ -169,15 +160,20 @@ No inference host or model ID is hardcoded. |---|---|---| | `FIREWORKS_BASE_URL` | Remote policies | Evaluator-provided OpenAI-compatible base URL | | `FIREWORKS_API_KEY` | Remote policies | Evaluator-provided bearer token | -| `ALLOWED_MODELS` | When supplied by evaluator | Strict JSON-array or comma/semicolon/whitespace-separated model allowlist | +| `ALLOWED_MODELS` | Remote policies | Official comma-separated allowlist; tolerant development parsing also accepts JSON/other separators | | `FALLTHROUGH_GEMMA_MODEL` | If Gemma cannot be inferred | Exact allowed Gemma model ID | | `FALLTHROUGH_POLICY` | No | `hybrid-baseline`, `gemma-only`, or `deterministic-only` | | `FALLTHROUGH_ROUTE_TABLE` | No | Validated measured-policy JSON used by `hybrid-baseline` | -| `FALLTHROUGH_LOCAL_BACKEND` | No | `none` or optional in-process `vllm`; default `none` | -| `FALLTHROUGH_LOCAL_MODEL` | Local experiments | Candidate local path/model identifier; never selected by default policy | +| `FALLTHROUGH_PACKAGED_ROUTE_TABLE` | Image build only | Image-owned measured table; must be paired with evidence | +| `FALLTHROUGH_PACKAGED_POLICY_EVIDENCE` | Image build only | Fail-closed evidence sidecar binding exact models, source, prompt policy, accuracy, and complete usage | +| `FALLTHROUGH_LOCAL_BACKEND` | No | `none` or optional owned-process `llama_cpp`; default `none` | +| `FALLTHROUGH_LOCAL_MODEL` | Local experiments | Existing baked `.gguf` path; never selected by default policy | +| `FALLTHROUGH_LOCAL_THREADS` | No | llama.cpp CPU threads, `1` or `2`; default `2` | +| `FALLTHROUGH_LOCAL_CONTEXT_SIZE` | No | Measured llama.cpp context, `128`-`4096`; default `1024` | | `FALLTHROUGH_GLOBAL_TIMEOUT_SECONDS` | No | Whole-batch deadline; default `270` | +| `FALLTHROUGH_TASK_TIMEOUT_SECONDS` | No | Shared wall budget for all routes on one task; default `29`, always below `30` | | `FALLTHROUGH_OUTPUT_RESERVE_SECONDS` | No | Deadline time retained for output; default `2` | -| `FALLTHROUGH_REQUEST_TIMEOUT_SECONDS` | No | Per-route wall and HTTP read timeout; default `30` | +| `FALLTHROUGH_REQUEST_TIMEOUT_SECONDS` | No | Per-route wall and HTTP read timeout; default `25`, capped by task budget | | `FALLTHROUGH_CONNECT_TIMEOUT_SECONDS` | No | Fireworks connect timeout; default `5` | | `FALLTHROUGH_TRANSPORT_RETRIES` | No | Transport/timeout/5xx retries; default `1` | | `FALLTHROUGH_FORMAT_RETRIES` | No | Validation-gated corrective retries per remote call; default `1`, `0` disables | @@ -186,11 +182,19 @@ No inference host or model ID is hardcoded. | `FALLTHROUGH_OUTPUT_PATH` | No | Output JSON path; otherwise stdout | | `FALLTHROUGH_TRACE_PATH` | No | Trace JSONL path; otherwise stderr | +The current published candidates (2026-07-11) are `minimax-m3`, +`kimi-k2p7-code`, `gemma-4-31b-it`, `gemma-4-26b-a4b-it`, and +`gemma-4-31b-it-nvfp4`. They are documented benchmark candidates, not production +constants; the injected `ALLOWED_MODELS` remains authoritative. + If `FALLTHROUGH_GEMMA_MODEL` is absent, the first allowlisted model whose ID -contains `gemma` is selected. When `ALLOWED_MODELS` is present it is enforced -strictly, including an explicitly empty list; an explicit Gemma model must occur -in it. The client appends `/chat/completions` only when the injected base URL does -not already end with that path. +contains `gemma` is selected. Remote execution requires a declared, nonempty +`ALLOWED_MODELS`; an explicit Gemma model must identify Gemma and occur in it. +The client records a proxy-reported model name separately from the requested +allowlisted ID; unresolved mismatches block policy evidence rather than discarding a potentially valid scored answer. It appends +`/chat/completions` only when the injected base URL does not already end with that +path. Legacy allowlist aliases are intentionally ignored: only the official +`ALLOWED_MODELS` variable can authorize a model. Only transport errors, timeouts, HTTP 5xx responses, and objectively format-invalid answers (validation-gated, once by default) are retried. @@ -198,29 +202,33 @@ Successful but merely uncertain answers are never repeated. API-reported prompt and completion usage—including usage reported by retried 5xx responses—is recorded; character-count estimates are never substituted. If a cancelled or ambiguous transport attempt has no authoritative usage response, its usage is marked -unknown and excluded from measured token means rather than treated as zero. +unknown. The aggregate's primary total/mean/median become `null` until every row has +authoritative usage; a separately labelled known-only mean and observed-minimum +lower bound remain diagnostic and must not be used to promote a route. `FIREWORKS_BASE_URL` is the only outbound inference destination. The runtime has no clients for OpenAI, Anthropic, Gemini, hosted Hugging Face, Groq, Together, -OpenRouter, external Ollama/vLLM, search, RAG, or vector services. Do not replace -the injected URL with Fireworks' normal public endpoint. +OpenRouter, external Ollama/vLLM, search, RAG, or vector services. There is no +endpoint default or fallback. Development may explicitly set the standard +Fireworks URL; the scored container uses the harness-injected URL unchanged. ### Measured route tables -A route table can select `deterministic`, canonical `fw_gemma` for the selected -Gemma model, or `fw_model:` for any other model. -The eval harness uses the same canonical Gemma name even when Gemma is included -through `--route fw_generic`. Every configured route is resolved and checked for -task-kind support before inference; unavailable models fail closed instead of -being silently skipped. Example: +A measured route table can select `deterministic` or +`fw_model:` for every remote candidate, including +Gemma. The `fw_gemma` alias is late-bound to the first allowlisted Gemma and is +therefore reserved for the unmeasured baseline and explicit `gemma-only` mode; +serialized measured tables reject it. This prevents allowlist reordering from +silently applying evidence to another Gemma. Every configured route is resolved +and checked before inference; unavailable models fail closed. Example: ```json { "routes": { - "arithmetic": ["deterministic", "fw_gemma"], - "sentiment": ["fw_model:accounts/example/models/cheap-model", "fw_gemma"] + "arithmetic": ["deterministic", "fw_model:evaluator-measured-model"], + "sentiment": ["fw_model:evaluator-measured-model"] }, - "unknown_routes": ["fw_gemma"] + "unknown_routes": ["fw_model:evaluator-measured-model"] } ``` @@ -231,8 +239,9 @@ claim that a particular routing policy has already been measured. ## Run locally Python 3.10 through 3.12 is supported. The probed AMD notebook pod uses Python -3.10.12; the current container base remains Python 3.12 until the organizer's -final container contract is confirmed and tested on a Docker-capable host. +3.10.12, and the submitted image uses the current official +`python:3.10.20-slim-bookworm` tag so the scored artifact exercises the minimum +supported interpreter line. ```bash python -m venv .venv @@ -241,7 +250,7 @@ python -m pip install -e '.[dev]' export FIREWORKS_BASE_URL='https://evaluator-provided.example/v1' export FIREWORKS_API_KEY='...' -export ALLOWED_MODELS='["evaluator-provided-gemma-model"]' +export ALLOWED_MODELS='evaluator-provided-gemma-model' fallthrough --input tasks.json --output results.json --trace traces/run.jsonl ``` @@ -249,16 +258,16 @@ fallthrough --input tasks.json --output results.json --trace traces/run.jsonl An offline arithmetic smoke test needs no credentials: ```bash -printf '[{"id":"m1","prompt":"Calculate 19 * 7."}]' \ +printf '[{"task_id":"m1","prompt":"Calculate 19 * 7."}]' \ | FALLTHROUGH_POLICY=deterministic-only fallthrough ``` ### Development credentials (`.env`) Keep development credentials in a `.env` file at the repository root. It is -listed in both `.gitignore` and `.dockerignore`, so it can never reach a commit -or the container build context; keep it that way and never bake credentials -into an image or environment defaults. +listed in both `.gitignore` and `.dockerignore`, which prevents ordinary adds and +container builds from including it. Never force-add or bake credentials into an +image or environment defaults. The runtime deliberately reads only real process environment variables — there is no dotenv dependency and no automatic `.env` loading, because the scored @@ -278,19 +287,18 @@ Get-Content .env | Where-Object { $_ -match '^\s*[^#]' } | ForEach-Object { ``` `FIREWORKS_API_KEY` alone is not enough to test remotely: `FIREWORKS_BASE_URL` -has no default and must be the evaluator/hackathon-provided endpoint. Do not -put the public Fireworks API URL in `.env` — the compliance addendum forbids -substituting it anywhere, including local development runs. A key that has -transited chat, email, or a ticket should be treated as semi-exposed and -rotated once the event ends. +and a nonempty `ALLOWED_MODELS` have no defaults. The scored runtime uses the +injected hackathon endpoint. Development may explicitly configure the standard +Fireworks URL and an explicit development allowlist; it is never an automatic +fallback or baked image value. Offline development uses local mocks. Rotate +any key that transits chat, email, or a ticket immediately. -## AMD notebook pod and local shadow evaluation +## AMD notebook pod, GGUF export, and local shadow evaluation -The probed development pod is Python 3.10.12 with ROCm, an available 48 GiB -`gfx1100` GPU, and vLLM 0.16 development packages already installed. These are -development capabilities, not assumptions about the final scoring container. -PyTorch and vLLM therefore remain optional lazy imports and are not added to the -normal package or scored-image dependency lock. +The probed development pod is Python 3.10.12 with ROCm and an available 48 GiB +`gfx1100` GPU. It is training/export infrastructure, not the inference runtime. +All route evidence must use the final quantized GGUF through the same llama.cpp +backend intended for the image, under the final CPU/RAM limits. Probe the current environment without printing credential values: @@ -298,54 +306,77 @@ Probe the current environment without printing credential values: python scripts/probe_environment.py ``` -Run a bounded load/generation smoke test with an existing local path or cached -model identifier: +Install the pinned optional binding in the experiment environment, then run a +bounded smoke test with an existing local GGUF: ```bash -FALLTHROUGH_LOCAL_BACKEND=vllm \ -python scripts/smoke_test_vllm.py /workspace/models/candidate \ - --model-label candidate-a --max-model-len 2048 --max-tokens 16 +python -m pip install -e '.[local]' +FALLTHROUGH_LOCAL_BACKEND=llama_cpp \ +python scripts/smoke_test_llama_cpp.py /workspace/models/candidate.gguf \ + --model-label candidate-a --context-size 1024 --threads 2 --max-tokens 16 ``` -Network model download is disabled unconditionally. Stage audited weights -outside the repository before starting Fallthrough; only safetensors weights are -accepted and remote model code is always disabled. Existing local paths are -content-hashed before load. A locally cached model identifier additionally -requires `--model-revision` with its full 40-hex commit; this binds the experiment -identity but does not claim to be a content digest of cache files. The optional -`--download-dir` argument names a local cache directory despite vLLM's parameter -name; it never enables network access. +There is no model identifier, Hub, URL, or download path. The backend accepts one +existing, non-symlink GGUF file and validates its header and size; the smoke and +benchmark tools content-hash that artifact before load. The backend forces CPU +execution, caps threads at two, and owns a worker process that is terminated when +load or generation exceeds its wall budget. + +A bounded, little-endian GGUF v2/v3 inspector rejects split checkpoints, sums +stored tensor dimensions as `header_tensor_element_count`, reads the declared +`general.file_type`, and records the tensor-type distribution. After loading, +the worker obtains the parameter count and file type through llama.cpp's public +model APIs; disagreement with the header fails the load. Only an active nominal +four-bit-or-smaller file class whose expected tensor type holds a majority of +stored elements can populate complete promotion evidence. This nominal class is +not a claim that every tensor uses exactly that many physical bits, and unknown, +removed, mixed, or missing declarations remain non-promotable. Benchmark the same development cases used by Fireworks routes: ```bash -FALLTHROUGH_LOCAL_BACKEND=vllm \ -FALLTHROUGH_LOCAL_MODEL=/workspace/models/candidate \ +FALLTHROUGH_LOCAL_BACKEND=llama_cpp \ +FALLTHROUGH_LOCAL_MODEL=/workspace/models/candidate.gguf \ python -m eval.benchmark_local_routes eval/datasets/smoke-arithmetic.json \ --model-label candidate-a --output eval/results/local-candidate.jsonl ``` -Every local observation includes answer, correctness, latency, failure and -validation status, generated-token counts, and observed VRAM where available. -It also records the vLLM configuration, version, GPU architecture/VRAM, a stable +Every local observation includes answer, recomputable scorer evidence, end-to-end +parent wall latency, separate backend latency, failure and validation status, +generated-token counts, and total cgroup peak memory where available (falling +back to worker peak RSS). +It also records the llama.cpp configuration/version, CPU/cgroup identity, a stable experiment fingerprint, `shadow_mode=true`, `used_as_final_answer=false`, and zero Fireworks tokens. Route aggregation includes that fingerprint, so results -from materially different configurations are not silently combined. Supply a +from materially different code, artifact, timeout, or hardware configurations +are not silently combined. Supply a single stable `--model-label` when comparing the same checkpoint across filesystem locations. Replacing a checkpoint in place changes its content identity automatically; use a new label as well to keep reports human-readable. -VRAM is the highest observed global usage on device 0 and can include other -processes; benchmark on an otherwise idle pod. Measured experiments are limited -to one GPU. This tool never modifies the production policy or evaluator output. -The synchronous vLLM backend is deliberately not registered as a scored route: -thread cancellation cannot guarantee that GPU generation stops, and these -in-process development commands have no internal hard-kill boundary. On Linux, -wrap experiments with the system `timeout --kill-after=...` command when a wall -limit is required. Enable a local final-answer route only after evaluator -hardware is known, weights are baked offline, held-out accuracy wins, and -cancellation is proven with an owned process or validated asynchronous abort -path. +This tool never modifies the production policy or evaluator output. The owned +worker provides a native-code kill boundary, but `local_model` is still absent +from the scored registry until the exact image/artifact wins on untouched +holdout while meeting startup, per-task, batch, memory, and image-size gates. +Aggregates expose `constraint_evidence_complete=false` until the exact GGUF run +records a content identity, sub-60-second total startup, cgroup-wide peak below +4,000,000,000 bytes, a runtime/header-matched 2B-3B parameter count, a single-file checkpoint, +and a cross-checked nominal quantization class at four bits or smaller. These +artifact facts do not replace exact-image holdout accuracy, deadline, batch, or +compressed-image measurements. Shadow accuracy alone is therefore never +promotion-ready evidence. + +The Participant Guide describes 2B-3B four-bit models as safe sizing guidance, +not an official eligibility band. The narrower parameter/quantization check +above is Fallthrough's current conservative project promotion gate from the +final architecture; the actual judging constraints remain measured memory, +startup, task/batch runtime, image size, and accuracy. + +The default startup field covers in-process checkpoint hashing plus model load; +it is diagnostic, not cold-container proof. Only pass +`--cold-startup-time-seconds` with a wall time measured by the external +Docker/constrained-run harness. The promotion predicate requires that explicit +provenance and otherwise remains false. On the pod, GitHub must use its pod-specific SSH-over-443 setup: @@ -353,23 +384,38 @@ On the pod, GitHub must use its pod-specific SSH-over-443 setup: git remote set-url origin git@github.com:latch-lab/fallthrough.git ``` -Never copy the pod SSH key, Hugging Face cache, vLLM cache, model weights, or -generated benchmark artifacts into this repository. Remove the registered key +Never copy the pod SSH key, training caches, source checkpoints, model weights, +or generated benchmark artifacts into this repository. Remove the registered key when the pod is retired. Docker is unavailable on the pod; build and test the final image on a separate Docker-capable host. ## Build and run the container -The image uses Python 3.12, a pinned runtime/build dependency closure in +The image uses Python 3.10.20, a pinned runtime/build dependency closure in `requirements.lock`, and no runtime package/model downloads. It intentionally keeps the evaluator image simple and does not assume bind-mounted output -directories are writable by a custom UID. The organizer's required base-image -digest remains an official-runtime detail to confirm before submission. +directories are writable by a custom UID. ```bash -docker build --no-cache -t fallthrough . +# Load FIREWORKS_API_KEY, FIREWORKS_BASE_URL, and ALLOWED_MODELS into this shell. +sh scripts/build_container.sh fallthrough +FALLTHROUGH_INPUT_DIR="$PWD/input" FALLTHROUGH_OUTPUT_DIR="$PWD/output" \ + sh scripts/run_container_constrained.sh fallthrough ``` +The build command verifies `amd64`, runs a cold offline container/task under a +strict 59-second timeout, and checks a gzipped local image stream below +10,000,000,000 bytes (a conservative decimal interpretation of 10 GB); +verify the published registry artifact again because registry compression can +differ. The constrained command forwards only the named Fireworks/routing variables, +enforces 4,000,000,000 bytes with no extra swap and 2 vCPU, removes stale output +first, validates the exact list schema and ID coverage, and forcibly terminates a batch before ten minutes. A real local +model's separate startup/load evidence must also remain below 60 seconds. +The runner forwards the audited routing/timeout/local variables listed in the +script. To test a measured route table, set +`FALLTHROUGH_ROUTE_TABLE_HOST=/absolute/host/path/routes.json`; it is mounted +read-only at a fixed in-container path. + Run with mounted evaluator files: ```bash @@ -382,8 +428,8 @@ docker run --rm \ --trace /data/trace.jsonl ``` -The addendum's conceptual `/input/tasks.json` to `/output/results.json` layout is -also supported directly. When `/input/tasks.json` exists, those input/output paths +The official `/input/tasks.json` to `/output/results.json` layout is supported +directly. When `/input/tasks.json` exists, those input/output paths are auto-selected even without CLI flags: ```bash @@ -409,6 +455,11 @@ docker run --rm -i --network=none \ < eval/datasets/smoke-arithmetic.json ``` +Before using a submission slot, push the exact tag to a public registry, pull it +from a clean machine, and verify the registry manifest contains `linux/amd64`. +The organizer limits Track 1 to 10 submissions per hour per team; an +infrastructure error is not a reason to resubmit repeatedly. + ## Route evaluation Proxy datasets use a development-only schema documented in @@ -429,15 +480,15 @@ regression evidence, not sufficient by itself for router calibration. Benchmark the deterministic route without network access: ```bash -python -m eval.run_routes eval/datasets/development.json \ +python -m eval.run_routes eval/datasets/math-development.jsonl \ --route deterministic \ --output eval/results/deterministic.jsonl ``` -Benchmark Gemma and every other allowlisted model: +Benchmark every exact allowlisted model, including every Gemma candidate: ```bash -python -m eval.run_routes eval/datasets/development.json \ +python -m eval.run_routes path/to/multi-family-development.json \ --route all \ --output eval/results/all-routes.jsonl ``` @@ -447,20 +498,84 @@ Optionally add a Gemma self-consistency arm (`--self-consistency 3` runs a route selection; the scored runtime never registers it and the default route table never selects it. -Aggregate accuracy, known token usage, unknown-usage rate, invalid-output rate, -latency, and failures: +Aggregate transparent local proxy accuracy, total and mean known token usage, unknown-usage rate, +invalid-output rate, latency, startup/load time, total observed memory, and failures. Name every +intended candidate with `--require-route`; this makes a wholly absent route an +error rather than invisible evidence: ```bash python -m eval.compare_routes eval/results/all-routes.jsonl \ - --output eval/results/route-table.json -python -m eval.report eval/results/route-table.json \ - --output eval/results/route-table.md + --require-route deterministic \ + --require-route 'fw_model:' \ + --output eval/results/route-comparison.json +python -m eval.report eval/results/route-comparison.json \ + --output eval/results/route-comparison.md +``` + +Repeat `--require-route` for every `fw_model:` and local candidate. +Fireworks and local JSONL files from the same dataset may be supplied together. +The comparator rejects empty/truncated coverage, mixed splits or run identities, +contradictory booleans, and correctness values it cannot reproduce from the +recorded answer, target, and scorer. It also requires every local row to identify +one content-hashed GGUF and verifies the recorded run/context hashes. The +aggregate is evidence, not an executable +`FALLTHROUGH_ROUTE_TABLE`; choose and review that policy separately. +After authoring an exact-model route table, replay its first-success fallthrough +order over those same full-coverage observations. This counts failed attempts, +conservatively enforces the shared task deadline, and reports policy-level local +proxy accuracy plus total Fireworks tokens: + +```bash +python -m eval.evaluate_policy eval/results/all-routes.jsonl \ + --route-table policy/measured.json \ + --output eval/results/measured-policy.json ``` -No holdout accuracy, qualification-rate, token table, or Gemma ablation is -claimed in this README yet because those experiments have not been run. The -observation and aggregation infrastructure is present specifically so those -results can be measured rather than invented. +Any unknown usage makes the primary policy total `null`; the observed minimum is +reported separately and must not be used to claim a winning token total. +Likewise, local exact/text scorers do not reproduce the official LLM judge. +Policy promotion still requires official evaluator results or another audited +semantic/functional rubric; code cases cannot silently default to exact matching. +Route execution timeouts begin after the benchmark semaphore is acquired so +queue position cannot reduce a model's answer budget. Recorded wall latency still +includes queue time, and the aggregate reports a sub-30-second deadline violation +rate; concurrency is bound into the run fingerprint. + +### Current development evidence + +The repository now includes two original 40-case sets with five cases in each +required family. The calibration fingerprint is +`c441691870d1601900e12a47f3a4354ce627275f6e9d1876a5da9fc886fde7fe`; the +separate audit split fingerprint is +`596f368d1228d61a074f19fe9cd422a494c2f0da81d8e72d2f4cdf65cd5f006e`. +Neither contains public evaluator prompt fingerprints. + +On the explicit public development endpoint, the fully qualified Kimi +development model completed the 40-case calibration run without a route or +format failure. Surface-form proxy scoring was 32/40 because five valid summary +paraphrases, one punctuation-only logic answer, and two functionally equivalent +code answers were outside finite accepted-string lists. A conservative manual +semantic/functional audit scored 39/40; the remaining Bash answer met the stated +task but omitted the defensive `cp --` form. The deterministic-first composition +selected a zero-token route for five cases and used 2,234 reported Fireworks +tokens, versus 2,400 for Kimi-only on the same run. + +The organizer's ten retired Track 1 validation examples were also run as a +development smoke (they are not committed as hidden/holdout data). Kimi answered +all ten without runtime or format failure and satisfied the published rubrics on +manual review, using 1,534 reported Fireworks tokens. MiniMax, at the same +accuracy-preserving output ceilings, completed four of ten and used 4,177 +tokens; its other six answers hit a length or validation failure. These results +support Kimi as the current development hypothesis and the selective +conciseness rule, but they do **not** activate a release route table: the public +fully qualified IDs are not the evaluator's short injected IDs, and none of the +three Gemma endpoints is available on the public development account. + +No official qualification rate, exact-proxy model comparison, or Gemma ablation +is claimed. The image therefore remains `baseline-unmeasured` until current +source is evaluated through the injected proxy and a reviewed policy/evidence +pair is packaged. Public-development observations are ignored build artifacts +and are never rebound to another model ID. ## Verification @@ -488,9 +603,9 @@ ordering, and the offline evaluation harness. semantic answer cache, remote judge, or prompt memorization is included. - Code validators parse and compile candidate Python; they do not execute arbitrary evaluator code in the baseline runtime. -- A future local model must execute inside this same image with weights already - available offline; no developer-machine, remote Ollama/vLLM, cloud pod, or - assumed evaluator GPU counts as local inference. +- A selected local model must execute through the baked GGUF/llama.cpp worker in + this image; no developer-machine, remote Ollama/vLLM, cloud pod, runtime model + download, or assumed evaluator GPU counts as local inference. ## License diff --git a/docs/amd-pod-environment.md b/docs/amd-pod-environment.md index dcad254..9177e63 100644 --- a/docs/amd-pod-environment.md +++ b/docs/amd-pod-environment.md @@ -13,17 +13,18 @@ Probe date: 2026-07-11. | Docker | unavailable | | Fireworks environment | not injected on this pod | -The pod is a local-model feasibility and measurement environment. It is not the -final scoring container, and its GPU must not become a hard routing assumption. -Use it for vLLM compatibility, latency/VRAM measurements, and shadow-mode route -traces. Keep candidate selection measurement-driven. +The pod is training, fine-tuning, export, and quantization infrastructure. It is +not the final scoring container, and its GPU must not become a routing +assumption. Accuracy/latency evidence used for final policy decisions must come +from the exported GGUF through Fallthrough's llama.cpp worker under 4 GB RAM and +2 vCPU. Repository workflow: ```bash cd /workspace/fallthrough git pull -python -m pip install -e . +python -m pip install -e '.[dev]' pytest python scripts/probe_environment.py ``` @@ -37,14 +38,10 @@ git remote set-url origin git@github.com:latch-lab/fallthrough.git Do not print, copy, or commit that key. Remove it from GitHub when the pod is retired. The final scoring container must not assume the pod SSH configuration. -The optional backend follows vLLM's documented offline `LLM`/`SamplingParams` -surface and uses `LLM.generate(..., use_tqdm=False)`. It is lazy, defaults off, -forces Hugging Face/Transformers offline mode, accepts safetensors only, and is -shadow-only until final runtime compatibility and held-out accuracy are known. -There is no download opt-in. Pre-stage audited weights outside the repository. -Local paths are content-hashed before load; cached identifiers require a full -40-hex immutable revision. Recorded experiment fingerprints include that -checkpoint identity, the non-secret model label, generation policy, vLLM -configuration/version, and detected GPU architecture/VRAM so incomparable runs -do not share an aggregation key. Observed VRAM is global device-0 use and should -be measured on an otherwise idle pod. See the official [vLLM 0.16 offline LLM API](https://docs.vllm.ai/en/v0.16.0/api/vllm/entrypoints/llm/). +The Fallthrough local backend uses a content-hashed GGUF file through a pinned +`llama-cpp-python`/llama.cpp runtime. It is lazy, CPU-only, defaults off, has no +model identifier or download path, and remains shadow-only until held-out and +container-constraint gates pass. Its owned worker is killed on a native +generation timeout. Recorded fingerprints include artifact, generation, +runtime, prompt-policy, CPU/cgroup, and memory evidence. See the official +[llama-cpp-python API](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/). diff --git a/docs/developer-handoff-2026-07-11.md b/docs/developer-handoff-2026-07-11.md index 1748ef1..915b692 100644 --- a/docs/developer-handoff-2026-07-11.md +++ b/docs/developer-handoff-2026-07-11.md @@ -1,5 +1,9 @@ # Fallthrough developer handoff +> Historical handoff: the vLLM plan in this document is superseded by +> [final-runtime-architecture.md](final-runtime-architecture.md). Preserve this +> file only as an audit trail of the earlier implementation state. + Date: 2026-07-11 Repository: `C:\Users\hepha\PycharmProjects\fallthrough` Remote: `git@github.com:latch-lab/fallthrough.git` @@ -21,8 +25,9 @@ Allowed final-answer paths are deterministic in-container computation, a model executing inside the scored container, or the evaluator-provided Fireworks endpoint. No other remote inference, search, RAG, vector, or information service is allowed. Outbound access is deny-by-default except for the injected -`FIREWORKS_BASE_URL`. Fireworks models must come from `ALLOWED_MODELS` when that -variable is supplied. Never substitute the public Fireworks API URL. +`FIREWORKS_BASE_URL`. Fireworks models must come from `ALLOWED_MODELS`. Never +invent an endpoint fallback; development may explicitly supply the standard +Fireworks URL, while scoring uses the harness-injected URL unchanged. Local inference counts as zero scored Fireworks tokens. Gemma must do meaningful work through Fireworks to qualify for the additional Gemma award; decorative @@ -72,7 +77,7 @@ Adopted concepts/components include: models; - classifier, label, arithmetic, and code-fence improvements. -Rejected components include any behavior that used a public Fireworks URL, +Rejected components include any behavior that invented a Fireworks URL fallback, bypassed an empty allowlist, weakened token accounting or deadlines, introduced an evaluator-incompatible service, or otherwise violated the compliance addendum. SymPy/mpmath were removed to reduce the runtime footprint. diff --git a/docs/final-runtime-architecture.md b/docs/final-runtime-architecture.md new file mode 100644 index 0000000..d27e060 --- /dev/null +++ b/docs/final-runtime-architecture.md @@ -0,0 +1,132 @@ +# Final Track 1 runtime architecture + +Recorded: 2026-07-11. This decision supersedes the earlier vLLM/local-backend +plan. The compliance addendum still governs allowed answer-generation paths and +network access. + +## Final grader contract + +The scored container is `linux/amd64` with: + +- 4 GB RAM; +- 2 vCPU; +- startup under 60 seconds; +- under 30 seconds per task; +- 10-minute total runtime; +- compressed image under 10 GB. + +It reads a top-level list of `{task_id, prompt}` objects from +`/input/tasks.json`, writes a top-level list of `{task_id, answer}` objects to +`/output/results.json`, and exits zero after a successful non-interactive batch. +Answers are in English. The public image must have a pullable `linux/amd64` +manifest. The harness injects `FIREWORKS_API_KEY`, `FIREWORKS_BASE_URL`, and +`ALLOWED_MODELS`. + +## One hybrid implementation + +The only eligible route types are: + +1. deterministic in-container computation; +2. an optional compact local GGUF model through llama.cpp; +3. an allowlisted model through the injected Fireworks endpoint. + +Routing is measured by task family. Neither local-first nor Fireworks-first is a +policy. Deterministic routes commit only when their answer can be proved. A +local call is useful only when its held-out accuracy/token benefit exceeds its +latency, memory, startup, image-size, and timeout risk. + +The local model is optional. If no candidate improves the probability of +clearing the accuracy gate with fewer Fireworks tokens under the exact final +constraints, ship deterministic plus Fireworks and omit llama.cpp/model weight +overhead. + +## Local model boundary + +Fallthrough has one local generative engine: llama.cpp. vLLM is development-pod +history, not a Fallthrough runtime backend. + +Any shipped local model must be: + +- GGUF; +- approximately 2B-3B parameters; +- four-bit quantized or smaller; +- CPU-compatible; +- baked into the image; +- proven under 4 GB RAM and 2 vCPU. + +The Participant Guide presents 2B-3B four-bit as safe sizing guidance, not an +official eligibility range. The narrower size/quantization bullets above remain +Fallthrough's conservative project promotion gate until exact-image evidence +justifies revisiting them. + +Artifact qualification is fail-closed. A bounded little-endian GGUF v2/v3 +inspector rejects shards, counts stored tensor elements, reads the declared +nominal majority file type, and records tensor-type distribution. The owned +worker then cross-checks parameter count and file type through llama.cpp's +loaded-model APIs. Unknown/removed types, inconsistent tensor majorities, or a +runtime/header mismatch cannot satisfy the promotion gate; header facts alone +do not prove accuracy or exact-container fitness. + +Task timing evidence uses parent end-to-end wall latency, with backend latency +retained separately. Cold-startup qualification must come from an external +container wall-clock measurement; an in-process load timer is diagnostic only. + +The AMD pod may train or fine-tune with PyTorch/ROCm. Routing evidence must be +collected after merge/export, GGUF conversion, quantization, and execution via +the exact llama.cpp integration intended for the image. Pod GPU performance is +not final-runtime evidence. + +Local inference remains disabled unless a valid baked GGUF exists and a measured +route table explicitly selects it. + +## Fireworks boundary + +The scored runtime has no endpoint default or public fallback. Every call uses +the injected `FIREWORKS_BASE_URL`; every selected model must appear in the +runtime `ALLOWED_MODELS`. Missing or invalid configuration fails closed, and +actual prompt/completion usage is recorded. + +Measured policies use `fw_model:` for every remote +candidate, including Gemma. The late-bound `fw_gemma` alias is reserved for the +unmeasured baseline and `gemma-only` convenience mode so allowlist reordering +cannot silently apply measured evidence to a different model. + +Development may use local HTTP mocks, an explicitly configured standard +Fireworks endpoint, or the injected hackathon endpoint. The runtime has no +endpoint default or fallback, and scoring uses the injected URL unchanged. + +Gemma through Fireworks must perform genuine semantic or answer-generation work. +Other allowed Fireworks models remain eligible when measurements improve the +main Track 1 strategy. + +Local or deterministic final answers and a zero-Fireworks-call batch are valid. +The undisclosed accuracy gate is primary; qualifying policies are compared by +total proxy tokens, including added prompt tokens and generated completion +tokens. + +## Evidence required before policy changes + +For every task family and route candidate, record: + +- accuracy on calibration and untouched holdout cases; +- Fireworks prompt and completion tokens; +- latency and startup/load time; +- invalid-output and runtime-failure rates; +- cgroup-wide peak memory (worker RSS is diagnostic fallback only); +- the exact model artifact, runtime, prompt, generation, and hardware identity. + +Initial route expectations are hypotheses only: deterministic arithmetic/list +operations/supported logic, likely Fireworks for factual and unverifiable +semantic work, and measured comparisons elsewhere. + +## Delivery order + +1. Keep the current deterministic/Fireworks container healthy. +2. Obtain a first official evaluator result and Fireworks per-family baselines. +3. Evaluate compact GGUF candidates using the exact llama.cpp runtime. +4. Enable a local final-answer route only when constrained held-out measurements + justify it. +5. Build/test the final image with `--platform linux/amd64 --memory=4000000000 --cpus=2` + and verify all startup, task, batch, and compressed-image limits. +6. Push the exact tag, pull it from a clean machine, and verify the public + registry manifest contains `linux/amd64` before spending a submission slot. diff --git a/docs/track-1-official-contract.md b/docs/track-1-official-contract.md new file mode 100644 index 0000000..b75a914 --- /dev/null +++ b/docs/track-1-official-contract.md @@ -0,0 +1,135 @@ +# Track 1 official contract and source precedence + +Reviewed: 2026-07-11. Sources reviewed in full: the ten-page ACT II Participant +Guide, the judging FAQ/self-check document, and the general event FAQ supplied +to the team. The Participant Guide was also rendered page by page for visual +verification. + +## Source precedence + +Use the Track 1-specific Participant Guide and current published Track 1 model +list for the scored runtime. Use the judging self-check for answer-quality and +operational guidance. Treat the broad event FAQ as contextual when it conflicts +with those more specific, newer materials. + +This resolves two apparent conflicts: + +- the broad FAQ mentions only MiniMax and Kimi, while the current Track 1 list + below also contains three Gemma endpoints; +- the FAQ says judges do not need a live API endpoint. That means no persistent + web service is required; Track 1 still requires a public Docker image that the + batch evaluator can run. + +The scored runtime remains deny-by-default: outbound inference goes only to the +injected `FIREWORKS_BASE_URL`. For local development, the team may explicitly +set the standard Fireworks URL in `.env`; it is never a production default or +baked container value. + +## Exact evaluator contract + +The container reads `/input/tasks.json` as a top-level JSON array: + +```json +[ + {"task_id": "t1", "prompt": "..."}, + {"task_id": "t2", "prompt": "..."} +] +``` + +It writes `/output/results.json` before exit as a top-level JSON array: + +```json +[ + {"task_id": "t1", "answer": "..."}, + {"task_id": "t2", "answer": "..."} +] +``` + +There must be exactly one answer for every input ID. IDs must be preserved. The +output must be valid JSON and answers must be in English. Per-task failures +should be isolated wherever possible. A successful batch exits `0`; a process +or batch-level failure exits nonzero. + +The runtime limits are: + +- `linux/amd64`; +- 4 GB RAM and 2 vCPU; +- startup/ready in under 60 seconds; +- each task in under 30 seconds; +- complete batch in under 10 minutes; +- public, pullable image no larger than 10 GB compressed. + +Before submission, pull the exact public image tag on a clean machine and check +its registry manifest includes `linux/amd64`. Track 1 submissions are limited to +10 per hour per team; infrastructure errors should not trigger repeated queue +submissions. + +## Fireworks and current candidates + +The evaluator injects `FIREWORKS_API_KEY`, `FIREWORKS_BASE_URL`, and the +comma-separated `ALLOWED_MODELS`. All requests must use that base URL, and every requested model +must be an exact member of the runtime allowlist. Production code must not +contain a required launch-day model constant. + +The published Track 1 candidate list supplied on 2026-07-11 is: + +- `minimax-m3` +- `kimi-k2p7-code` +- `gemma-4-31b-it` +- `gemma-4-26b-a4b-it` +- `gemma-4-31b-it-nvfp4` + +This list is benchmark input, not runtime authority. `ALLOWED_MODELS` remains +authoritative and may change. Fallthrough benchmarks exact +`fw_model:` routes for all candidates. The late-bound +`fw_gemma` name is reserved for the unmeasured baseline and `gemma-only` +convenience policy; evidence-backed route tables must name the exact measured +route so allowlist reordering cannot silently change the model. + +Gemma through Fireworks must do meaningful semantic or answer-generation work +for the Gemma award. A decorative call is not acceptable. + +## Scoring + +An LLM judge first applies an accuracy gate. The threshold is not published and +must not be invented. Qualifying submissions are then ranked by ascending total +proxy tokens. Both added prompt/instruction tokens and generated completion +tokens count. + +Deterministic computation and local inference inside the scored container are +valid final-answer paths and consume zero scored Fireworks tokens. A run with +zero Fireworks calls may receive `ZERO_API_CALLS`; that is explicitly valid, not +a failure. This does not make local-only a mandatory strategy: routing must be +chosen from measured accuracy and total-token evidence. + +The guide describes 2B-3B four-bit local models as safe sizing guidance and warns +that 7B four-bit fills the RAM budget. It is not an official eligibility band. +Fallthrough currently retains the narrower 2B-3B/four-bit rule as a conservative +project promotion gate from the final runtime architecture; it must be described +as project policy, not a judging rule, and can be revisited only with exact-image +4 GB/2-vCPU evidence. + +## Task and answer-quality coverage + +The hidden set covers all eight families: factual knowledge, mathematical +reasoning, sentiment classification, summarisation, named-entity recognition, +code debugging, logical reasoning, and code generation. + +The public examples are retired scoring cases. They are useful development +regressions but are neither final cases nor holdout evidence. Do not memorize, +fingerprint, cache, or tune specifically to their wording. + +They establish general answer contracts that the runtime must preserve: + +- factual questions may require a concise explanation, not a bare label; +- multi-step math may require all requested values and enough working; +- sentiment may require a label plus a one-sentence reason, including mixed + nuance; +- summarisation sentence, bullet, and word-count constraints are strict; +- NER must include every entity with the requested type; +- every output must answer the actual prompt directly and completely. + +Accuracy comes before output-length tuning. Fallthrough therefore sends the +evaluator prompt verbatim by default, adds only missing trusted metadata +constraints, conditionally overrides an explicit non-English output request, and +avoids destructive post-processing of requested explanations. diff --git a/eval/benchmark_local_routes.py b/eval/benchmark_local_routes.py index cce164c..ccd8978 100644 --- a/eval/benchmark_local_routes.py +++ b/eval/benchmark_local_routes.py @@ -19,18 +19,32 @@ from dataclasses import asdict, dataclass from pathlib import Path -from eval.common import EvalCase, load_cases, score_answer +from eval.common import ( + SCORING_POLICY, + EvalCase, + benchmark_context_fingerprint, + dataset_fingerprint, + dataset_split, + load_cases, + observation_correct, + resolve_scorer, +) +from eval.constraints import local_constraint_evidence_complete from fallthrough.inference import ( DEFAULT_MAX_TOKENS_BY_KIND, + NORMALIZATION_POLICY, + PROMPT_POLICY, build_task_prompt, - normalize_code_answer, + canonicalize_answer, ) from fallthrough.local import ( CheckpointIdentity, + LlamaCppBackend, LocalBackend, - VllmBackend, + LocalBackendError, experiment_fingerprint, identify_checkpoint, + source_tree_fingerprint, stable_model_label, unverified_checkpoint_identity, ) @@ -45,6 +59,18 @@ class LocalRouteObservation: task_id: str task_kind: str task_family: str + declared_task_kind: str | None + classified_task_kind: str + dataset_fingerprint: str + dataset_split: str + dataset_task_count: int + implementation_sha256: str + shared_implementation_sha256: str + benchmark_context_fingerprint: str + declared_scorer: str + effective_scorer: str + scoring_policy: str + run_configuration: dict[str, object] route_name: str backend: str model: str @@ -62,6 +88,7 @@ class LocalRouteObservation: correct: bool correctness: bool latency_ms: float + backend_latency_ms: float | None failed: bool runtime_failure: bool output_valid: bool | None @@ -71,10 +98,13 @@ class LocalRouteObservation: local_prompt_tokens: int | None local_completion_tokens: int | None finish_reason: str | None - peak_vram_bytes: int | None - vram_measurement: str | None + peak_memory_bytes: int | None + memory_measurement: str | None load_time_seconds: float - load_peak_vram_bytes: int | None + load_peak_memory_bytes: int | None + startup_time_seconds: float | None + startup_measurement: str | None + constraint_evidence_complete: bool cumulative_logprob: float | None = None mean_logprob: float | None = None prompt_tokens: int = 0 @@ -108,9 +138,9 @@ def _generation_configuration( dict(DEFAULT_MAX_TOKENS_BY_KIND) if max_tokens is None else None ), "max_tokens_override": max_tokens, - "normalization_policy": "code-fence-v1", - "prompt_policy": "compact-task-prompt-v1", - "scoring_policy": "declared-local-scorer-v1", + "normalization_policy": NORMALIZATION_POLICY, + "prompt_policy": PROMPT_POLICY, + "scoring_policy": SCORING_POLICY, "temperature": temperature, } @@ -130,8 +160,6 @@ def _logprob_fields( def _case_kind(case: EvalCase) -> TaskKind: - if case.kind is not None: - return case.kind features = extract_features(case.task) return classify_task(case.task, features) @@ -154,6 +182,8 @@ def run_shadow_benchmark( model_label: str | None = None, checkpoint_identity: CheckpointIdentity | None = None, checkpoint_identity_time_seconds: float = 0.0, + startup_time_seconds: float | None = None, + startup_measurement: str | None = None, ) -> list[LocalRouteObservation]: """Generate sequential local observations without affecting runtime answers.""" @@ -166,20 +196,64 @@ def run_shadow_benchmark( ) configuration = _metadata_mapping(getattr(backend, "configuration", {})) generation_configuration = _generation_configuration(max_tokens, temperature) + dataset_id = dataset_fingerprint(cases) + split = dataset_split(cases) + dataset_count = len(cases) + common_source = Path(__file__).with_name("common.py") + shared_implementation_id = source_tree_fingerprint(common_source) + benchmark_context_id = benchmark_context_fingerprint( + dataset_id=dataset_id, + dataset_split_name=split, + dataset_task_count=dataset_count, + shared_implementation_sha256=shared_implementation_id, + ) + implementation_id = source_tree_fingerprint( + Path(__file__), + common_source, + Path(__file__).with_name("constraints.py"), + ) + generation_configuration["dataset_fingerprint"] = dataset_id + generation_configuration["dataset_split"] = split + generation_configuration["startup_measurement"] = startup_measurement + generation_configuration["startup_time_seconds"] = startup_time_seconds hardware = _metadata_mapping(getattr(backend, "hardware", {})) run_fingerprint = experiment_fingerprint( backend=backend.backend_name, model_label=experiment_model, checkpoint_identity_sha256=checkpoint.identity_sha256, + checkpoint_kind=checkpoint.kind, + checkpoint_file_count=checkpoint.file_count, + checkpoint_total_bytes=checkpoint.total_bytes, configuration={ "backend": configuration, "generation": generation_configuration, }, hardware=hardware, + implementation_sha256=implementation_id, + shared_implementation_sha256=shared_implementation_id, + benchmark_context_fingerprint=benchmark_context_id, ) - vram_measurement = getattr(backend, "vram_measurement", None) + memory_measurement = getattr(backend, "memory_measurement", None) + run_configuration: dict[str, object] = { + "backend": backend.backend_name, + "checkpoint_identity_sha256": checkpoint.identity_sha256, + "checkpoint_kind": checkpoint.kind, + "checkpoint_file_count": checkpoint.file_count, + "checkpoint_total_bytes": checkpoint.total_bytes, + "configuration": { + "backend": configuration, + "generation": generation_configuration, + }, + "execution_mode": "local_shadow", + "hardware": hardware, + "implementation_sha256": implementation_id, + "shared_implementation_sha256": shared_implementation_id, + "benchmark_context_fingerprint": benchmark_context_id, + "model_label": experiment_model, + } for case in cases: kind = _case_kind(case) + scorer = resolve_scorer(case, kind) started = time.perf_counter() try: generation = backend.generate_sync( @@ -188,11 +262,30 @@ def run_shadow_benchmark( temperature=temperature, ) except Exception as exc: + error_latency = exc.latency_ms if isinstance(exc, LocalBackendError) else None + error_peak_memory = ( + exc.peak_memory_bytes if isinstance(exc, LocalBackendError) else None + ) + error_memory_measurement = ( + exc.memory_measurement if isinstance(exc, LocalBackendError) else None + ) observations.append( LocalRouteObservation( task_id=case.task.id, task_kind=kind.value, - task_family=kind.value, + task_family=case.kind.value if case.kind is not None else kind.value, + declared_task_kind=case.kind.value if case.kind is not None else None, + classified_task_kind=kind.value, + dataset_fingerprint=dataset_id, + dataset_split=split, + dataset_task_count=dataset_count, + implementation_sha256=implementation_id, + shared_implementation_sha256=shared_implementation_id, + benchmark_context_fingerprint=benchmark_context_id, + declared_scorer=case.scorer, + effective_scorer=scorer, + scoring_policy=SCORING_POLICY, + run_configuration=run_configuration, route_name=_route_name( backend.backend_name, experiment_model, @@ -213,7 +306,11 @@ def run_shadow_benchmark( expected_answer=case.expected_answer, correct=False, correctness=False, - latency_ms=(time.perf_counter() - started) * 1000.0, + latency_ms=max( + (time.perf_counter() - started) * 1000.0, + error_latency or 0.0, + ), + backend_latency_ms=error_latency, failed=True, runtime_failure=True, output_valid=None, @@ -223,10 +320,13 @@ def run_shadow_benchmark( local_prompt_tokens=None, local_completion_tokens=None, finish_reason=None, - peak_vram_bytes=None, - vram_measurement=vram_measurement, + peak_memory_bytes=error_peak_memory, + memory_measurement=error_memory_measurement or memory_measurement, load_time_seconds=backend.load_time_seconds, - load_peak_vram_bytes=backend.load_peak_vram_bytes, + load_peak_memory_bytes=backend.load_peak_memory_bytes, + startup_time_seconds=startup_time_seconds, + startup_measurement=startup_measurement, + constraint_evidence_complete=False, error_code="local_generation_failed", error_type=type(exc).__name__, error_message="local generation failed", @@ -234,7 +334,7 @@ def run_shadow_benchmark( ) continue - answer = normalize_code_answer(case.task, kind, generation.text.strip()) + answer = canonicalize_answer(case.task, kind, generation.text.strip()) validation_reason: str | None = None invalid_output: bool | None = None validation_score: float | None = None @@ -252,29 +352,39 @@ def run_shadow_benchmark( if generation.finish_reason in {"length", "abort", "error", "content_filter"}: invalid_output = True validation_reason = f"incomplete generation: {generation.finish_reason}" - scorer = ( - "exact" - if case.scorer == "auto" and kind in {TaskKind.CODE_DEBUG, TaskKind.CODE_GENERATION} - else case.scorer - ) - correct = ( - bool(answer) - and invalid_output is not True - and score_answer( - answer, - case.expected_answer, - scorer, - ) + failed = not bool(answer) or invalid_output is True + correct = observation_correct( + answer, + case.expected_answer, + scorer, + failed=failed, + output_valid=(None if invalid_output is None else not invalid_output), ) cumulative_logprob, mean_logprob = _logprob_fields( generation.metadata, generation.token_count, ) + end_to_end_latency_ms = max( + (time.perf_counter() - started) * 1000.0, + generation.latency_ms, + ) observations.append( LocalRouteObservation( task_id=case.task.id, task_kind=kind.value, - task_family=kind.value, + task_family=case.kind.value if case.kind is not None else kind.value, + declared_task_kind=case.kind.value if case.kind is not None else None, + classified_task_kind=kind.value, + dataset_fingerprint=dataset_id, + dataset_split=split, + dataset_task_count=dataset_count, + implementation_sha256=implementation_id, + shared_implementation_sha256=shared_implementation_id, + benchmark_context_fingerprint=benchmark_context_id, + declared_scorer=case.scorer, + effective_scorer=scorer, + scoring_policy=SCORING_POLICY, + run_configuration=run_configuration, route_name=_route_name( backend.backend_name, experiment_model, @@ -295,8 +405,9 @@ def run_shadow_benchmark( expected_answer=case.expected_answer, correct=correct, correctness=correct, - latency_ms=generation.latency_ms, - failed=not bool(answer) or invalid_output is True, + latency_ms=end_to_end_latency_ms, + backend_latency_ms=generation.latency_ms, + failed=failed, runtime_failure=False, output_valid=(None if invalid_output is None else not invalid_output), invalid_output=invalid_output, @@ -305,14 +416,32 @@ def run_shadow_benchmark( local_prompt_tokens=generation.prompt_token_count, local_completion_tokens=generation.token_count, finish_reason=generation.finish_reason, - peak_vram_bytes=generation.peak_vram_bytes, - vram_measurement=( - generation.metadata.get("vram_measurement") - if isinstance(generation.metadata.get("vram_measurement"), str) - else vram_measurement + peak_memory_bytes=generation.peak_memory_bytes, + memory_measurement=( + generation.metadata.get("memory_measurement") + if isinstance(generation.metadata.get("memory_measurement"), str) + else memory_measurement ), load_time_seconds=backend.load_time_seconds, - load_peak_vram_bytes=backend.load_peak_vram_bytes, + load_peak_memory_bytes=backend.load_peak_memory_bytes, + startup_time_seconds=startup_time_seconds, + startup_measurement=startup_measurement, + constraint_evidence_complete=local_constraint_evidence_complete( + checkpoint_kind=checkpoint.kind, + checkpoint_identity_sha256=checkpoint.identity_sha256, + checkpoint_file_count=checkpoint.file_count, + checkpoint_total_bytes=checkpoint.total_bytes, + configuration=configuration, + startup_time_seconds=startup_time_seconds, + startup_measurement=startup_measurement, + memory_measurement=( + generation.metadata.get("memory_measurement") + if isinstance(generation.metadata.get("memory_measurement"), str) + else memory_measurement + ), + load_peak_memory_bytes=backend.load_peak_memory_bytes, + generation_peak_memory_bytes=generation.peak_memory_bytes, + ), cumulative_logprob=cumulative_logprob, mean_logprob=mean_logprob, ) @@ -332,22 +461,76 @@ def _load_failure_observations( hardware: dict[str, object], load_time_seconds: float, exc: BaseException, + implementation_id: str, + startup_time_seconds: float, + startup_measurement: str, ) -> list[LocalRouteObservation]: + dataset_id = dataset_fingerprint(cases) + split = dataset_split(cases) + dataset_count = len(cases) + common_source = Path(__file__).with_name("common.py") + shared_implementation_id = source_tree_fingerprint(common_source) + benchmark_context_id = benchmark_context_fingerprint( + dataset_id=dataset_id, + dataset_split_name=split, + dataset_task_count=dataset_count, + shared_implementation_sha256=shared_implementation_id, + ) + generation_configuration = dict(generation_configuration) + generation_configuration["dataset_fingerprint"] = dataset_id + generation_configuration["dataset_split"] = split + generation_configuration["startup_measurement"] = startup_measurement + generation_configuration["startup_time_seconds"] = startup_time_seconds run_fingerprint = experiment_fingerprint( backend=backend, model_label=model_label, checkpoint_identity_sha256=checkpoint.identity_sha256, + checkpoint_kind=checkpoint.kind, + checkpoint_file_count=checkpoint.file_count, + checkpoint_total_bytes=checkpoint.total_bytes, configuration={ "backend": configuration, "generation": generation_configuration, }, hardware=hardware, + implementation_sha256=implementation_id, + shared_implementation_sha256=shared_implementation_id, + benchmark_context_fingerprint=benchmark_context_id, ) + run_configuration: dict[str, object] = { + "backend": backend, + "checkpoint_identity_sha256": checkpoint.identity_sha256, + "checkpoint_kind": checkpoint.kind, + "checkpoint_file_count": checkpoint.file_count, + "checkpoint_total_bytes": checkpoint.total_bytes, + "configuration": { + "backend": configuration, + "generation": generation_configuration, + }, + "execution_mode": "local_shadow", + "hardware": hardware, + "implementation_sha256": implementation_id, + "shared_implementation_sha256": shared_implementation_id, + "benchmark_context_fingerprint": benchmark_context_id, + "model_label": model_label, + } return [ LocalRouteObservation( task_id=case.task.id, task_kind=_case_kind(case).value, - task_family=_case_kind(case).value, + task_family=case.kind.value if case.kind is not None else _case_kind(case).value, + declared_task_kind=case.kind.value if case.kind is not None else None, + classified_task_kind=_case_kind(case).value, + dataset_fingerprint=dataset_id, + dataset_split=split, + dataset_task_count=dataset_count, + implementation_sha256=implementation_id, + shared_implementation_sha256=shared_implementation_id, + benchmark_context_fingerprint=benchmark_context_id, + declared_scorer=case.scorer, + effective_scorer=resolve_scorer(case, _case_kind(case)), + scoring_policy=SCORING_POLICY, + run_configuration=run_configuration, route_name=_route_name(backend, model_label, run_fingerprint), backend=backend, model=model_label, @@ -365,6 +548,7 @@ def _load_failure_observations( correct=False, correctness=False, latency_ms=0.0, + backend_latency_ms=None, failed=True, runtime_failure=True, output_valid=None, @@ -374,10 +558,13 @@ def _load_failure_observations( local_prompt_tokens=None, local_completion_tokens=None, finish_reason=None, - peak_vram_bytes=None, - vram_measurement=None, + peak_memory_bytes=None, + memory_measurement=None, load_time_seconds=load_time_seconds, - load_peak_vram_bytes=None, + load_peak_memory_bytes=None, + startup_time_seconds=startup_time_seconds, + startup_measurement=startup_measurement, + constraint_evidence_complete=False, error_code="local_model_load_failed", error_type=type(exc).__name__, error_message="local model load failed", @@ -425,24 +612,21 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("dataset", type=Path) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--backend", choices=("none", "vllm")) + parser.add_argument("--backend", choices=("none", "llama_cpp")) parser.add_argument("--model") parser.add_argument("--model-label") - parser.add_argument( - "--model-revision", - type=_revision, - help="immutable hex commit required when --model is a cached identifier", - ) parser.add_argument("--expected-checkpoint-identity-sha256", type=_sha256) parser.add_argument("--limit", type=int) parser.add_argument("--max-tokens", type=_positive_int) - parser.add_argument("--max-model-len", type=_positive_int, default=2048) - parser.add_argument("--dtype", choices=("auto", "half", "float16", "bfloat16"), default="auto") - parser.add_argument("--gpu-memory-utilization", type=_utilization, default=0.82) - parser.add_argument("--enforce-eager", action="store_true") + parser.add_argument("--context-size", type=_context_size) + parser.add_argument("--threads", type=_threads) + parser.add_argument("--batch-size", type=_batch_size, default=128) + parser.add_argument("--load-timeout-seconds", type=_load_timeout, default=50.0) + parser.add_argument("--generation-timeout-seconds", type=_generation_timeout, default=25.0) parser.add_argument( - "--download-dir", - help="local vLLM/Hugging Face cache directory; network access remains disabled", + "--cold-startup-time-seconds", + type=_nonnegative_seconds, + help="Externally measured cold container-to-ready wall time", ) return parser.parse_args(argv) @@ -454,13 +638,71 @@ def _positive_int(value: str) -> int: return parsed -def _utilization(value: str) -> float: +def _bounded_int(value: str, minimum: int, maximum: int, label: str) -> int: + parsed = int(value) + if not minimum <= parsed <= maximum: + raise argparse.ArgumentTypeError(f"{label} must be between {minimum} and {maximum}") + return parsed + + +def _context_size(value: str) -> int: + return _bounded_int(value, 128, 4096, "context size") + + +def _threads(value: str) -> int: + return _bounded_int(value, 1, 2, "threads") + + +def _batch_size(value: str) -> int: + return _bounded_int(value, 1, 512, "batch size") + + +def _environment_bounded_int( + name: str, + default: int, + minimum: int, + maximum: int, +) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + return _bounded_int(raw, minimum, maximum, name) + except (ValueError, argparse.ArgumentTypeError) as exc: + raise SystemExit(str(exc)) from exc + + +def _bounded_float(value: str, maximum: float, label: str) -> float: + parsed = float(value) + if not math.isfinite(parsed) or not 0.0 < parsed <= maximum: + raise argparse.ArgumentTypeError(f"{label} must be finite, positive, and <= {maximum}") + return parsed + + +def _load_timeout(value: str) -> float: + return _bounded_float(value, 55.0, "load timeout") + + +def _generation_timeout(value: str) -> float: + return _bounded_float(value, 25.0, "generation timeout") + + +def _nonnegative_seconds(value: str) -> float: parsed = float(value) - if not math.isfinite(parsed) or not 0.0 < parsed <= 1.0: - raise argparse.ArgumentTypeError("must be finite and in (0, 1]") + if not math.isfinite(parsed) or parsed < 0: + raise argparse.ArgumentTypeError("must be finite and non-negative") return parsed +def _startup_evidence( + external_cold_seconds: float | None, + in_process_seconds: float, +) -> tuple[float, str]: + if external_cold_seconds is not None: + return external_cold_seconds, "external_cold_container_wall_seconds" + return in_process_seconds, "in_process_checkpoint_hash_and_model_load_seconds" + + def _sha256(value: str) -> str: normalized = value.casefold() if len(normalized) != 64 or any( @@ -470,15 +712,6 @@ def _sha256(value: str) -> str: return normalized -def _revision(value: str) -> str: - normalized = value.casefold() - if len(normalized) != 40 or any( - character not in "0123456789abcdef" for character in normalized - ): - raise argparse.ArgumentTypeError("must be a full 40-digit hexadecimal commit") - return normalized - - def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) cases = load_cases(args.dataset) @@ -486,19 +719,22 @@ def main(argv: Sequence[str] | None = None) -> int: if args.limit <= 0: raise SystemExit("--limit must be positive") cases = cases[: args.limit] + for case in cases: + resolve_scorer(case, _case_kind(case)) backend_name = (args.backend or os.getenv("FALLTHROUGH_LOCAL_BACKEND", "none")).lower() model = args.model or os.getenv("FALLTHROUGH_LOCAL_MODEL") if backend_name == "none": raise SystemExit( - "local backend is disabled; pass --backend vllm or set FALLTHROUGH_LOCAL_BACKEND=vllm" + "local backend is disabled; pass --backend llama_cpp or set " + "FALLTHROUGH_LOCAL_BACKEND=llama_cpp" ) - if backend_name != "vllm": + if backend_name != "llama_cpp": raise SystemExit(f"unsupported local backend: {backend_name}") if not model: raise SystemExit("--model or FALLTHROUGH_LOCAL_MODEL is required") identity_started = time.perf_counter() try: - checkpoint = identify_checkpoint(model, args.model_revision) + checkpoint = identify_checkpoint(model) experiment_model = stable_model_label( model, args.model_label, @@ -513,30 +749,46 @@ def main(argv: Sequence[str] | None = None) -> int: "checkpoint identity SHA-256 does not match --expected-checkpoint-identity-sha256" ) generation_configuration = _generation_configuration(args.max_tokens, 0.0) + implementation_id = source_tree_fingerprint( + Path(__file__), + Path(__file__).with_name("common.py"), + Path(__file__).with_name("constraints.py"), + ) + threads = args.threads or _environment_bounded_int("FALLTHROUGH_LOCAL_THREADS", 2, 1, 2) + context_size = args.context_size or _environment_bounded_int( + "FALLTHROUGH_LOCAL_CONTEXT_SIZE", 1024, 128, 4096 + ) + if args.batch_size > context_size: + raise SystemExit("--batch-size cannot exceed the context size") planned_configuration: dict[str, object] = { - "dtype": args.dtype, - "enforce_eager": args.enforce_eager, - "generation_config": "vllm", - "gpu_memory_utilization": args.gpu_memory_utilization, - "load_format": "safetensors", - "max_model_len": args.max_model_len, - "revision_bound": args.model_revision is not None, - "tensor_parallel_size": 1, - "vllm_version": None, + "batch_size": args.batch_size, + "chat_template_source": "gguf_metadata", + "context_size": context_size, + "llama_cpp_python_version": None, + "n_gpu_layers": 0, + "seed": 0, + "threads": threads, + "load_timeout_seconds": args.load_timeout_seconds, + "generation_timeout_seconds": args.generation_timeout_seconds, + "use_mlock": False, + "use_mmap": True, } load_started = time.perf_counter() try: - backend = VllmBackend.load( + backend = LlamaCppBackend.load( model, - dtype=args.dtype, - gpu_memory_utilization=args.gpu_memory_utilization, - max_model_len=args.max_model_len, - enforce_eager=args.enforce_eager, - download_dir=args.download_dir, - revision=args.model_revision, + threads=threads, + context_size=context_size, + batch_size=args.batch_size, + load_timeout_seconds=args.load_timeout_seconds, + generation_timeout_seconds=args.generation_timeout_seconds, ) except Exception as exc: + startup_seconds, startup_measurement = _startup_evidence( + args.cold_startup_time_seconds, + time.perf_counter() - identity_started, + ) observations = _load_failure_observations( cases, backend=backend_name, @@ -545,15 +797,22 @@ def main(argv: Sequence[str] | None = None) -> int: checkpoint_identity_time_seconds=checkpoint_identity_time_seconds, configuration=planned_configuration, generation_configuration=generation_configuration, - hardware={}, + hardware={"threads_used": threads}, load_time_seconds=time.perf_counter() - load_started, exc=exc, + implementation_id=implementation_id, + startup_time_seconds=startup_seconds, + startup_measurement=startup_measurement, ) _atomic_write_jsonl(args.output, observations) print(f"local backend load failed: {type(exc).__name__}", file=sys.stderr) return 1 try: + startup_seconds, startup_measurement = _startup_evidence( + args.cold_startup_time_seconds, + time.perf_counter() - identity_started, + ) observations = run_shadow_benchmark( cases, backend, @@ -561,6 +820,8 @@ def main(argv: Sequence[str] | None = None) -> int: model_label=experiment_model, checkpoint_identity=checkpoint, checkpoint_identity_time_seconds=checkpoint_identity_time_seconds, + startup_time_seconds=startup_seconds, + startup_measurement=startup_measurement, ) _atomic_write_jsonl(args.output, observations) finally: diff --git a/eval/common.py b/eval/common.py index 6ae0899..e616ac9 100644 --- a/eval/common.py +++ b/eval/common.py @@ -2,7 +2,9 @@ from __future__ import annotations +import hashlib import json +import re from dataclasses import dataclass, field from fractions import Fraction from pathlib import Path @@ -13,6 +15,8 @@ from fallthrough.validation.arithmetic import parse_numeric_answer _SCORERS = {"auto", "numeric", "json", "one_of", "exact", "casefold_text"} +SCORING_POLICY = "transparent-local-proxy-scorer-v3" +_GROUPED_NUMBER_RE = re.compile(r"^[+-]?\d{1,3}(?:,\d{3})+(?:\.\d+)?(?:[eE][+-]?\d+)?%?$") @dataclass(slots=True) @@ -69,6 +73,14 @@ def load_cases(path: Path) -> list[EvalCase]: f"evaluation record {index} has unknown scorer {scorer!r}; " f"expected one of {sorted(_SCORERS)}" ) + if scorer == "one_of" and not isinstance(record["expected_answer"], list): + raise ValueError(f"evaluation record {index} uses one_of without an array target") + if scorer == "auto" and kind in {TaskKind.CODE_DEBUG, TaskKind.CODE_GENERATION}: + raise ValueError( + f"evaluation record {index} is code and requires an explicit scorer; " + "use exact only for an exact-output contract, one_of for bounded accepted " + "answers, or add a transparent functional scorer" + ) cases.append( EvalCase( task=Task(id=identifier, prompt=prompt, metadata=task_metadata), @@ -84,6 +96,69 @@ def load_cases(path: Path) -> list[EvalCase]: return cases +def dataset_fingerprint(cases: list[EvalCase]) -> str: + """Hash the exact ordered evaluation contract without exposing it to routes.""" + + payload = [ + { + "declared_kind": case.kind.value if case.kind is not None else None, + "eval_metadata": case.eval_metadata, + "expected_answer": case.expected_answer, + "id": case.task.id, + "prompt": case.task.prompt, + "scorer": case.scorer, + "task_metadata": case.task.metadata, + } + for case in cases + ] + encoded = json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def dataset_split(cases: list[EvalCase]) -> str: + """Require one explicit split value (or uniformly unspecified data).""" + + splits = { + str(case.eval_metadata.get("split", "unspecified")).strip() or "unspecified" + for case in cases + } + if len(splits) != 1: + raise ValueError("evaluation dataset mixes multiple split labels") + return next(iter(splits)) + + +def benchmark_context_fingerprint( + *, + dataset_id: str, + dataset_split_name: str, + dataset_task_count: int, + shared_implementation_sha256: str, +) -> str: + """Identify conditions that must be identical across every candidate route.""" + + payload = { + "dataset_fingerprint": dataset_id, + "dataset_split": dataset_split_name, + "dataset_task_count": dataset_task_count, + "scoring_policy": SCORING_POLICY, + "shared_implementation_sha256": shared_implementation_sha256, + } + encoded = json.dumps( + payload, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def _normalize_text(value: Any) -> str: return " ".join(str(value).strip().casefold().split()) @@ -91,8 +166,13 @@ def _normalize_text(value: Any) -> str: def _numeric(value: Any) -> Fraction | None: if isinstance(value, bool | dict | list) or value is None: return None + text = str(value).strip() + if "," in text: + if _GROUPED_NUMBER_RE.fullmatch(text) is None: + return None + text = text.replace(",", "") try: - return parse_numeric_answer(str(value).strip().replace(",", "")) + return parse_numeric_answer(text) except (TypeError, ValueError, ZeroDivisionError): return None @@ -115,13 +195,67 @@ def score_answer(answer: str, expected: Any, scorer: str = "auto") -> bool: return actual_number == expected_number if scorer == "numeric": return False - if scorer in {"auto", "json"} and isinstance(expected, dict | list): + if scorer == "json" or (scorer == "auto" and isinstance(expected, dict)): try: actual_json = json.loads(answer) except json.JSONDecodeError: return False - return bool(actual_json == expected) - if scorer in {"auto", "one_of"} and isinstance(expected, list): + return _json_equal(actual_json, expected) + if scorer == "one_of": + if not isinstance(expected, list): + raise ValueError("one_of scorer requires an array target") + normalized = _normalize_text(answer) + return any(normalized == _normalize_text(item) for item in expected) + if scorer == "auto" and isinstance(expected, list): normalized = _normalize_text(answer) return any(normalized == _normalize_text(item) for item in expected) return _normalize_text(answer) == _normalize_text(expected) + + +def resolve_scorer(case: EvalCase, runtime_kind: TaskKind) -> str: + """Resolve automatic scorer rules from the dataset contract, not routing luck.""" + + if case.scorer != "auto": + return case.scorer + scoring_kind = case.kind or runtime_kind + if scoring_kind in {TaskKind.CODE_DEBUG, TaskKind.CODE_GENERATION}: + raise ValueError( + "code evaluation requires an explicit scorer; automatic exact matching is invalid" + ) + return "auto" + + +def observation_correct( + answer: str, + expected: Any, + scorer: str, + *, + failed: bool, + output_valid: bool | None, +) -> bool: + """Apply the cross-field correctness contract shared by writers/readers.""" + + return ( + not failed + and output_valid is not False + and bool(answer.strip()) + and score_answer(answer, expected, scorer) + ) + + +def _json_equal(actual: Any, expected: Any) -> bool: + """Compare JSON values without Python's ``True == 1`` coercion.""" + + if isinstance(actual, bool) or isinstance(expected, bool): + return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected + if isinstance(actual, int | float) and isinstance(expected, int | float): + return actual == expected + if isinstance(actual, dict) and isinstance(expected, dict): + return actual.keys() == expected.keys() and all( + _json_equal(actual[key], expected[key]) for key in actual + ) + if isinstance(actual, list) and isinstance(expected, list): + return len(actual) == len(expected) and all( + _json_equal(left, right) for left, right in zip(actual, expected, strict=True) + ) + return type(actual) is type(expected) and actual == expected diff --git a/eval/compare_routes.py b/eval/compare_routes.py index 3300275..3bf9090 100644 --- a/eval/compare_routes.py +++ b/eval/compare_routes.py @@ -4,22 +4,50 @@ from __future__ import annotations import argparse +import hashlib import json +import math import statistics from collections import defaultdict from pathlib import Path from typing import Any +from eval.common import SCORING_POLICY, benchmark_context_fingerprint, observation_correct +from eval.constraints import local_constraint_evidence_complete + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("observations", nargs="+", type=Path) parser.add_argument("--output", type=Path, help="Write aggregate JSON; default is stdout") + parser.add_argument( + "--require-route", + action="append", + default=[], + help="Route name that must be present; repeat for every intended candidate", + ) return parser.parse_args() -def aggregate(paths: list[Path]) -> dict[str, dict[str, dict[str, Any]]]: +def aggregate( + paths: list[Path], + *, + required_routes: set[str] | None = None, +) -> dict[str, dict[str, dict[str, Any]]]: grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + seen: set[tuple[str, str, str, str]] = set() + dataset_ids: set[str] = set() + route_runs: dict[tuple[str, str], set[str]] = defaultdict(set) + route_implementations: dict[tuple[str, str], set[str]] = defaultdict(set) + route_configurations: dict[tuple[str, str], set[str]] = defaultdict(set) + coverage: dict[tuple[str, str], dict[str, set[str]]] = defaultdict(lambda: defaultdict(set)) + route_coverage: dict[tuple[str, str], set[str]] = defaultdict(set) + dataset_task_counts: dict[str, set[int]] = defaultdict(set) + dataset_splits: dict[str, set[str]] = defaultdict(set) + dataset_routes: dict[str, set[str]] = defaultdict(set) + dataset_shared_implementations: dict[str, set[str]] = defaultdict(set) + dataset_benchmark_contexts: dict[str, set[str]] = defaultdict(set) + task_contracts: dict[tuple[str, str], str] = {} for path in paths: for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): if not line.strip(): @@ -28,22 +56,219 @@ def aggregate(paths: list[Path]) -> dict[str, dict[str, dict[str, Any]]]: row = json.loads(line) except json.JSONDecodeError as exc: raise ValueError(f"{path}:{line_number}: invalid JSON") from exc - grouped[(str(row["task_kind"]), str(row["route_name"]))].append(row) + _validate_row(row, path, line_number) + dataset_id = str(row["dataset_fingerprint"]) + route = str(row["route_name"]) + run_id = str(row["run_fingerprint"]) + task_id = str(row["task_id"]) + kind = str(row.get("task_family") or row["task_kind"]) + duplicate_key = (dataset_id, run_id, route, task_id) + if duplicate_key in seen: + raise ValueError(f"{path}:{line_number}: duplicate route/task observation") + seen.add(duplicate_key) + dataset_ids.add(dataset_id) + dataset_task_counts[dataset_id].add(int(row["dataset_task_count"])) + dataset_splits[dataset_id].add(str(row["dataset_split"])) + dataset_routes[dataset_id].add(route) + dataset_shared_implementations[dataset_id].add(str(row["shared_implementation_sha256"])) + dataset_benchmark_contexts[dataset_id].add(str(row["benchmark_context_fingerprint"])) + route_runs[(dataset_id, route)].add(run_id) + route_implementations[(dataset_id, route)].add(str(row["implementation_sha256"])) + route_configurations[(dataset_id, route)].add( + json.dumps( + row["run_configuration"], + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + ) + coverage[(dataset_id, kind)][route].add(task_id) + route_coverage[(dataset_id, route)].add(task_id) + contract = json.dumps( + { + "classified_task_kind": row["classified_task_kind"], + "declared_scorer": row["declared_scorer"], + "effective_scorer": row["effective_scorer"], + "expected_answer": row["expected_answer"], + "task_family": row["task_family"], + }, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + contract_key = (dataset_id, task_id) + prior_contract = task_contracts.setdefault(contract_key, contract) + if prior_contract != contract: + raise ValueError(f"{path}:{line_number}: inconsistent task evidence contract") + grouped[(kind, route)].append(row) + + if not grouped: + raise ValueError("no route observations were found") + if len(dataset_ids) > 1: + raise ValueError("cannot aggregate different dataset fingerprints in one report") + for dataset_id in dataset_ids: + counts = dataset_task_counts[dataset_id] + if len(counts) != 1: + raise ValueError(f"dataset {dataset_id} has inconsistent task counts") + expected_count = next(iter(counts)) + if len(dataset_splits[dataset_id]) != 1: + raise ValueError(f"dataset {dataset_id} has mixed split labels") + if len(dataset_shared_implementations[dataset_id]) != 1: + raise ValueError(f"dataset {dataset_id} has mixed shared implementations") + if len(dataset_benchmark_contexts[dataset_id]) != 1: + raise ValueError(f"dataset {dataset_id} has mixed benchmark contexts") + for route in dataset_routes[dataset_id]: + if len(route_coverage[(dataset_id, route)]) != expected_count: + raise ValueError( + f"route {route!r} has incomplete dataset coverage for {dataset_id}" + ) + expected_task_ids: set[str] | None = None + for route in sorted(dataset_routes[dataset_id]): + task_ids = route_coverage[(dataset_id, route)] + if expected_task_ids is None: + expected_task_ids = task_ids + elif task_ids != expected_task_ids: + raise ValueError(f"routes do not cover the same task IDs in dataset {dataset_id}") + missing_routes = (required_routes or set()) - dataset_routes[dataset_id] + if missing_routes: + raise ValueError("required routes are absent: " + ", ".join(sorted(missing_routes))) + for (dataset_id, route), run_ids in route_runs.items(): + if len(run_ids) != 1: + raise ValueError(f"route {route!r} has mixed run fingerprints for dataset {dataset_id}") + if len(route_implementations[(dataset_id, route)]) != 1: + raise ValueError(f"route {route!r} has mixed implementation identities") + if len(route_configurations[(dataset_id, route)]) != 1: + raise ValueError(f"route {route!r} has mixed run configurations") + for (dataset_id, kind), route_tasks in coverage.items(): + expected: set[str] | None = None + for route, task_ids in sorted(route_tasks.items()): + if expected is None: + expected = task_ids + elif task_ids != expected: + raise ValueError( + f"incomplete task coverage for {kind!r}/{route!r} in dataset {dataset_id}" + ) result: dict[str, dict[str, dict[str, Any]]] = {} for (kind, route), rows in sorted(grouped.items()): - known_token_rows = [row for row in rows if _usage_known(row)] + known_token_rows = [row for row in rows if row["usage_known"]] tokens = [_scored_tokens(row) for row in known_token_rows] + observed_minimum_tokens = [_scored_tokens(row) for row in rows] latencies = [float(row.get("latency_ms", 0.0)) for row in rows] - validated = [row for row in rows if row.get("output_valid") is not None] + validated = [row for row in rows if row["output_valid"] is not None] + memory = [ + int(row["peak_memory_bytes"]) + for row in rows + if isinstance(row.get("peak_memory_bytes"), int) + and not isinstance(row.get("peak_memory_bytes"), bool) + ] + load_memory = [ + int(row["load_peak_memory_bytes"]) + for row in rows + if isinstance(row.get("load_peak_memory_bytes"), int) + and not isinstance(row.get("load_peak_memory_bytes"), bool) + ] + load_times = [ + float(row["load_time_seconds"]) + for row in rows + if isinstance(row.get("load_time_seconds"), int | float) + and not isinstance(row.get("load_time_seconds"), bool) + ] + startup_times = [ + float(row["startup_time_seconds"]) + for row in rows + if isinstance(row.get("startup_time_seconds"), int | float) + and not isinstance(row.get("startup_time_seconds"), bool) + ] + startup_measurements = sorted( + { + value + for row in rows + if isinstance((value := row.get("startup_measurement")), str) and value + } + ) + checkpoint_identity_times = [ + float(row["checkpoint_identity_time_seconds"]) + for row in rows + if isinstance(row.get("checkpoint_identity_time_seconds"), int | float) + and not isinstance(row.get("checkpoint_identity_time_seconds"), bool) + ] + all_memory = memory + load_memory + served_models = sorted( + { + served + for row in rows + if isinstance(row.get("route_features"), dict) + and isinstance((served := row["route_features"].get("served_model")), str) + and served + } + ) + model_identity_flags = [ + features.get("configured_model") == features.get("served_model") + for row in rows + if isinstance((features := row.get("route_features")), dict) + and isinstance(features.get("configured_model"), str) + and bool(features.get("configured_model")) + and isinstance(features.get("served_model"), str) + and bool(features.get("served_model")) + ] + constraint_flags = [row.get("constraint_evidence_complete") for row in rows] + has_constraint_evidence = any(isinstance(value, bool) for value in constraint_flags) + run_id = str(rows[0]["run_fingerprint"]) result.setdefault(kind, {})[route] = { + "dataset_fingerprint": str(rows[0]["dataset_fingerprint"]), + "dataset_split": str(rows[0]["dataset_split"]), + "dataset_task_count": int(rows[0]["dataset_task_count"]), + "run_fingerprint": run_id, + "implementation_sha256": str(rows[0]["implementation_sha256"]), + "shared_implementation_sha256": str(rows[0]["shared_implementation_sha256"]), + "benchmark_context_fingerprint": str(rows[0]["benchmark_context_fingerprint"]), + "scoring_policy": str(rows[0]["scoring_policy"]), + "run_configuration": rows[0]["run_configuration"], "count": len(rows), - "accuracy": sum(bool(row.get("correct")) for row in rows) / len(rows), - "mean_fireworks_tokens": statistics.fmean(tokens) if tokens else None, - "median_fireworks_tokens": statistics.median(tokens) if tokens else None, + "proxy_accuracy": sum(row["correct"] for row in rows) / len(rows), + "accuracy": sum(row["correct"] for row in rows) / len(rows), + "accuracy_semantics": "transparent_local_proxy_not_official_llm_judge", + "total_fireworks_tokens": (sum(tokens) if len(known_token_rows) == len(rows) else None), + "total_fireworks_tokens_known_usage": sum(tokens), + "total_observed_minimum_fireworks_tokens": sum(observed_minimum_tokens), + "mean_fireworks_tokens": ( + statistics.fmean(tokens) if len(known_token_rows) == len(rows) else None + ), + "median_fireworks_tokens": ( + statistics.median(tokens) if len(known_token_rows) == len(rows) else None + ), + "mean_fireworks_tokens_known_usage": (statistics.fmean(tokens) if tokens else None), + "median_fireworks_tokens_known_usage": (statistics.median(tokens) if tokens else None), + "mean_observed_minimum_fireworks_tokens": statistics.fmean(observed_minimum_tokens), "usage_unknown_rate": 1.0 - (len(known_token_rows) / len(rows)), + "token_metrics_complete": len(known_token_rows) == len(rows), "mean_latency_ms": statistics.fmean(latencies), - "failure_rate": sum(bool(row.get("failed")) for row in rows) / len(rows), + "task_deadline_violation_rate": sum(latency >= 30_000.0 for latency in latencies) + / len(latencies), + "failure_rate": sum(row["failed"] for row in rows) / len(rows), + "load_time_seconds": max(load_times) if load_times else None, + "checkpoint_identity_time_seconds": ( + max(checkpoint_identity_times) if checkpoint_identity_times else None + ), + "startup_time_seconds": max(startup_times) if startup_times else None, + "startup_measurements": startup_measurements, + "load_peak_memory_bytes": max(load_memory) if load_memory else None, + "generation_peak_memory_bytes": max(memory) if memory else None, + "peak_memory_bytes": max(all_memory) if all_memory else None, + "served_models": served_models, + "model_identity_match_rate": ( + sum(model_identity_flags) / len(model_identity_flags) + if model_identity_flags + else None + ), + "constraint_evidence_complete": ( + all(value is True for value in constraint_flags) + if has_constraint_evidence + else None + ), "invalid_output_rate": ( sum(row.get("output_valid") is False for row in validated) / len(validated) if validated @@ -57,17 +282,272 @@ def _scored_tokens(row: dict[str, Any]) -> int: return int(row.get("prompt_tokens", 0)) + int(row.get("completion_tokens", 0)) -def _usage_known(row: dict[str, Any]) -> bool: - explicit = row.get("usage_known") - if isinstance(explicit, bool): - return explicit - route = str(row.get("route_name", "")) - return not (route.startswith("fw_") and bool(row.get("failed"))) +def _json_values_equal(left: object, right: object) -> bool: + try: + return json.dumps( + left, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) == json.dumps( + right, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + except (TypeError, ValueError): + return False + + +def _validate_row(row: object, path: Path, line_number: int) -> None: + prefix = f"{path}:{line_number}" + if not isinstance(row, dict): + raise ValueError(f"{prefix}: observation is not an object") + for key in ( + "task_id", + "task_kind", + "route_name", + "dataset_fingerprint", + "dataset_split", + "run_fingerprint", + "task_family", + "classified_task_kind", + "implementation_sha256", + "shared_implementation_sha256", + "benchmark_context_fingerprint", + "declared_scorer", + "effective_scorer", + "scoring_policy", + ): + value = row.get(key) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{prefix}: {key} must be nonblank text") + for key in ("correct", "failed", "usage_known"): + if not isinstance(row.get(key), bool): + raise ValueError(f"{prefix}: {key} must be a boolean") + constraint_evidence = row.get("constraint_evidence_complete") + if constraint_evidence is not None and not isinstance(constraint_evidence, bool): + raise ValueError(f"{prefix}: constraint_evidence_complete must be boolean or null") + output_valid = row.get("output_valid") + if output_valid is not None and not isinstance(output_valid, bool): + raise ValueError(f"{prefix}: output_valid must be boolean or null") + if row["correct"] and row["failed"]: + raise ValueError(f"{prefix}: a failed observation cannot be correct") + if output_valid is False and not row["failed"]: + raise ValueError(f"{prefix}: invalid output must be marked failed") + if output_valid is False and row["correct"]: + raise ValueError(f"{prefix}: invalid output cannot be correct") + if not isinstance(row.get("answer"), str): + raise ValueError(f"{prefix}: answer must be text") + if "expected_answer" not in row: + raise ValueError(f"{prefix}: expected_answer is required") + if row["scoring_policy"] != SCORING_POLICY: + raise ValueError(f"{prefix}: unsupported scoring_policy") + if not isinstance(row.get("run_configuration"), dict): + raise ValueError(f"{prefix}: run_configuration must be an object") + try: + encoded_run_configuration = json.dumps( + row["run_configuration"], + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise ValueError(f"{prefix}: run_configuration must contain finite JSON values") from exc + if hashlib.sha256(encoded_run_configuration).hexdigest() != row["run_fingerprint"]: + raise ValueError(f"{prefix}: run_fingerprint does not match run_configuration") + try: + recomputed = observation_correct( + row["answer"], + row["expected_answer"], + row["effective_scorer"], + failed=row["failed"], + output_valid=output_valid, + ) + except ValueError as exc: + raise ValueError(f"{prefix}: invalid scorer evidence") from exc + if recomputed is not row["correct"]: + raise ValueError(f"{prefix}: recorded correctness does not match scorer evidence") + dataset_count = row.get("dataset_task_count") + if isinstance(dataset_count, bool) or not isinstance(dataset_count, int) or dataset_count <= 0: + raise ValueError(f"{prefix}: dataset_task_count must be a positive integer") + if row["dataset_split"] == "unspecified": + raise ValueError(f"{prefix}: dataset_split must be explicit for comparison evidence") + expected_benchmark_context = benchmark_context_fingerprint( + dataset_id=row["dataset_fingerprint"], + dataset_split_name=row["dataset_split"], + dataset_task_count=dataset_count, + shared_implementation_sha256=row["shared_implementation_sha256"], + ) + if expected_benchmark_context != row["benchmark_context_fingerprint"]: + raise ValueError(f"{prefix}: benchmark_context_fingerprint is inconsistent") + for key in ( + "implementation_sha256", + "shared_implementation_sha256", + "benchmark_context_fingerprint", + ): + if row["run_configuration"].get(key) != row[key]: + raise ValueError(f"{prefix}: run_configuration disagrees with {key}") + for key in ( + "dataset_fingerprint", + "run_fingerprint", + "implementation_sha256", + "shared_implementation_sha256", + "benchmark_context_fingerprint", + ): + value = row[key] + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"{prefix}: {key} must be a lowercase SHA-256") + for key in ("prompt_tokens", "completion_tokens"): + value = row.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{prefix}: {key} must be a non-negative integer") + latency = row.get("latency_ms") + if ( + isinstance(latency, bool) + or not isinstance(latency, int | float) + or not math.isfinite(latency) + or latency < 0 + ): + raise ValueError(f"{prefix}: latency_ms must be finite and non-negative") + memory = row.get("peak_memory_bytes") + if memory is not None and ( + isinstance(memory, bool) or not isinstance(memory, int) or memory < 0 + ): + raise ValueError(f"{prefix}: peak_memory_bytes must be a non-negative integer or null") + load_memory = row.get("load_peak_memory_bytes") + if load_memory is not None and ( + isinstance(load_memory, bool) or not isinstance(load_memory, int) or load_memory < 0 + ): + raise ValueError(f"{prefix}: load_peak_memory_bytes must be a non-negative integer or null") + load_time = row.get("load_time_seconds") + if load_time is not None and ( + isinstance(load_time, bool) + or not isinstance(load_time, int | float) + or not math.isfinite(load_time) + or load_time < 0 + ): + raise ValueError(f"{prefix}: load_time_seconds must be finite and non-negative or null") + for key in ("checkpoint_identity_time_seconds", "startup_time_seconds"): + value = row.get(key) + if value is not None and ( + isinstance(value, bool) + or not isinstance(value, int | float) + or not math.isfinite(value) + or value < 0 + ): + raise ValueError(f"{prefix}: {key} must be finite and non-negative or null") + startup_measurement = row.get("startup_measurement") + if startup_measurement is not None and ( + not isinstance(startup_measurement, str) or not startup_measurement.strip() + ): + raise ValueError(f"{prefix}: startup_measurement must be nonblank text or null") + is_local_evidence = ( + row["route_name"].startswith("local_") + or row.get("shadow_mode") is True + or row["run_configuration"].get("execution_mode") == "local_shadow" + ) + if is_local_evidence: + if not row["route_name"].startswith("local_"): + raise ValueError(f"{prefix}: local evidence requires a local route name") + for key in ("backend", "model"): + if not isinstance(row.get(key), str) or not row[key].strip(): + raise ValueError(f"{prefix}: local {key} must be nonblank text") + if row.get("checkpoint_kind") != "local_gguf_sha256": + raise ValueError(f"{prefix}: local evidence requires a content-verified GGUF") + checkpoint_sha = row.get("checkpoint_identity_sha256") + if ( + not isinstance(checkpoint_sha, str) + or len(checkpoint_sha) != 64 + or any(character not in "0123456789abcdef" for character in checkpoint_sha) + ): + raise ValueError(f"{prefix}: local checkpoint identity must be a SHA-256") + checkpoint_file_count = row.get("checkpoint_file_count") + if ( + isinstance(checkpoint_file_count, bool) + or not isinstance(checkpoint_file_count, int) + or checkpoint_file_count != 1 + ): + raise ValueError(f"{prefix}: local checkpoint evidence must identify one GGUF file") + checkpoint_bytes = row.get("checkpoint_total_bytes") + if ( + isinstance(checkpoint_bytes, bool) + or not isinstance(checkpoint_bytes, int) + or checkpoint_bytes <= 4 + ): + raise ValueError(f"{prefix}: local checkpoint size must be recorded") + run_configuration = row["run_configuration"] + nested_configuration = run_configuration.get("configuration") + if not isinstance(nested_configuration, dict): + raise ValueError(f"{prefix}: local run configuration is incomplete") + for recorded_key, run_value in ( + ("backend_configuration", nested_configuration.get("backend")), + ("generation_configuration", nested_configuration.get("generation")), + ("hardware", run_configuration.get("hardware")), + ): + if not _json_values_equal(row.get(recorded_key), run_value): + raise ValueError(f"{prefix}: local {recorded_key} disagrees with run_configuration") + generation_configuration = row["generation_configuration"] + if ( + not isinstance(generation_configuration, dict) + or not _json_values_equal( + generation_configuration.get("startup_time_seconds"), + row.get("startup_time_seconds"), + ) + or generation_configuration.get("startup_measurement") != startup_measurement + ): + raise ValueError(f"{prefix}: local startup evidence disagrees with run_configuration") + if ( + run_configuration.get("execution_mode") != "local_shadow" + or run_configuration.get("backend") != row["backend"] + or run_configuration.get("model_label") != row["model"] + or run_configuration.get("checkpoint_identity_sha256") != checkpoint_sha + or run_configuration.get("checkpoint_kind") != row.get("checkpoint_kind") + or run_configuration.get("checkpoint_file_count") != checkpoint_file_count + or run_configuration.get("checkpoint_total_bytes") != checkpoint_bytes + ): + raise ValueError(f"{prefix}: local identity disagrees with run_configuration") + canonical_route_name = f"local_{row['backend']}:{row['model']}:{row['run_fingerprint']}" + if row["route_name"] != canonical_route_name: + raise ValueError(f"{prefix}: local route name is not canonical") + if row.get("shadow_mode") is not True or row.get("used_as_final_answer") is not False: + raise ValueError(f"{prefix}: local comparison evidence must remain shadow-only") + for key in ( + "prompt_tokens", + "completion_tokens", + "fireworks_prompt_tokens", + "fireworks_completion_tokens", + ): + if row.get(key) != 0 or isinstance(row.get(key), bool): + raise ValueError(f"{prefix}: local evidence must record zero {key}") + if row.get("usage_known") is not True: + raise ValueError(f"{prefix}: local zero-token usage must be known") + if constraint_evidence is True and ( + row["failed"] + or not local_constraint_evidence_complete( + checkpoint_kind=row.get("checkpoint_kind"), + checkpoint_identity_sha256=checkpoint_sha, + checkpoint_file_count=row.get("checkpoint_file_count"), + checkpoint_total_bytes=checkpoint_bytes, + configuration=row.get("backend_configuration"), + startup_time_seconds=row.get("startup_time_seconds"), + startup_measurement=row.get("startup_measurement"), + memory_measurement=row.get("memory_measurement"), + load_peak_memory_bytes=row.get("load_peak_memory_bytes"), + generation_peak_memory_bytes=row.get("peak_memory_bytes"), + ) + ): + raise ValueError(f"{prefix}: claimed local constraint evidence is incomplete") + elif constraint_evidence is True: + raise ValueError(f"{prefix}: only local observations can claim constraint evidence") def main() -> int: args = parse_args() - payload = aggregate(args.observations) + payload = aggregate(args.observations, required_routes=set(args.require_route)) text = json.dumps(payload, indent=2, sort_keys=True) + "\n" if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) diff --git a/eval/constraints.py b/eval/constraints.py new file mode 100644 index 0000000..04110f3 --- /dev/null +++ b/eval/constraints.py @@ -0,0 +1,117 @@ +"""Fail-closed evidence predicates shared by benchmark writers and readers.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping + +from fallthrough.local.gguf_metadata import ( + nominal_quantization_bits, + tensor_type_evidence_consistent, +) + +_FINAL_MEMORY_LIMIT_BYTES = 4_000_000_000 +_MIN_LOCAL_PARAMETERS = 2_000_000_000 +_MAX_LOCAL_PARAMETERS = 3_000_000_000 + + +def local_constraint_evidence_complete( + *, + checkpoint_kind: object, + checkpoint_identity_sha256: object, + checkpoint_file_count: object, + checkpoint_total_bytes: object, + configuration: object, + startup_time_seconds: object, + startup_measurement: object, + memory_measurement: object, + load_peak_memory_bytes: object, + generation_peak_memory_bytes: object, +) -> bool: + """Return whether one successful local row proves the promotion prerequisites. + + A generation-time cgroup peak is mandatory because that cumulative value + covers model load plus inference. Worker RSS is useful diagnostics but is + not proof that the whole 4 GB container stayed within its limit. + """ + + if not isinstance(configuration, Mapping): + return False + parameter_count = configuration.get("parameter_count") + header_parameter_count = configuration.get("header_tensor_element_count") + runtime_parameter_count = configuration.get("runtime_parameter_count") + file_type = configuration.get("gguf_file_type") + runtime_file_type = configuration.get("runtime_file_type") + quantization_bits = configuration.get("nominal_quantization_bits") + expected_quantization_bits = nominal_quantization_bits(file_type) + recomputed_tensor_consistency = tensor_type_evidence_consistent( + file_type, + configuration.get("tensor_type_parameter_counts"), + parameter_count, + ) + if ( + checkpoint_kind != "local_gguf_sha256" + or not _is_sha256(checkpoint_identity_sha256) + or isinstance(checkpoint_file_count, bool) + or not isinstance(checkpoint_file_count, int) + or checkpoint_file_count != 1 + or isinstance(checkpoint_total_bytes, bool) + or not isinstance(checkpoint_total_bytes, int) + or checkpoint_total_bytes <= 4 + or isinstance(startup_time_seconds, bool) + or not isinstance(startup_time_seconds, int | float) + or not math.isfinite(startup_time_seconds) + or not 0 <= startup_time_seconds < 60 + or startup_measurement != "external_cold_container_wall_seconds" + or memory_measurement != "container_cgroup_peak_memory_bytes" + or not _valid_peak(generation_peak_memory_bytes) + or isinstance(parameter_count, bool) + or not isinstance(parameter_count, int) + or not _MIN_LOCAL_PARAMETERS <= parameter_count <= _MAX_LOCAL_PARAMETERS + or isinstance(header_parameter_count, bool) + or not isinstance(header_parameter_count, int) + or isinstance(runtime_parameter_count, bool) + or not isinstance(runtime_parameter_count, int) + or parameter_count != header_parameter_count + or parameter_count != runtime_parameter_count + or configuration.get("parameter_count_crosscheck") != "match" + or isinstance(file_type, bool) + or not isinstance(file_type, int) + or isinstance(runtime_file_type, bool) + or not isinstance(runtime_file_type, int) + or file_type != runtime_file_type + or configuration.get("file_type_crosscheck") != "match" + or isinstance(configuration.get("split_count"), bool) + or configuration.get("split_count") != 1 + or configuration.get("gguf_endianness") != "little" + or isinstance(configuration.get("gguf_version"), bool) + or configuration.get("gguf_version") not in {2, 3} + or configuration.get("tensor_type_consistent") is not True + or recomputed_tensor_consistency is not True + or expected_quantization_bits is None + or isinstance(quantization_bits, bool) + or not isinstance(quantization_bits, int | float) + or not math.isfinite(quantization_bits) + or not 0 < quantization_bits <= 4 + or quantization_bits != expected_quantization_bits + ): + return False + if load_peak_memory_bytes is not None and not _valid_peak(load_peak_memory_bytes): + return False + assert isinstance(generation_peak_memory_bytes, int) + peaks: list[int] = [generation_peak_memory_bytes] + if isinstance(load_peak_memory_bytes, int): + peaks.append(load_peak_memory_bytes) + return max(peaks) < _FINAL_MEMORY_LIMIT_BYTES + + +def _valid_peak(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _is_sha256(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) diff --git a/eval/datasets/README.md b/eval/datasets/README.md index 853bc49..1f5a9ee 100644 --- a/eval/datasets/README.md +++ b/eval/datasets/README.md @@ -20,14 +20,44 @@ Keep development, calibration, and untouched holdout records in distinct files. Large generated results belong in `eval/results/`, which is gitignored. Do not put official evaluator prompts or memorized answer tables here. +## Versioned Track 1 proxy sets + +`calibration-v1.json` and `holdout-v1.json` are original, hand-authored proxy +sets. They contain 40 cases each: five cases for each of the eight Track 1 task +families. They were written for this repository and are not copied, paraphrased, +or fingerprinted from public or current evaluator questions. + +Use `calibration-v1.json` to compare routes and select thresholds. Treat +`holdout-v1.json` as a final confirmation set: do not inspect individual +holdout misses and then tune prompts, route rules, accepted-answer lists, or +model selection against them. A new version and a new untouched holdout are +required after any such tuning. + +Both files intentionally exercise classification and output contracts as well +as answer quality. Their declared family distribution is balanced, every case +has an explicit split and provenance marker, and every code case declares a +bounded `one_of` scorer. Run them through `eval/run_routes.py` so scoring also +requires a successful route and structurally valid output. + `task_metadata` is passed to the candidate task and is reserved for contract information such as allowed labels or output shape. `eval_metadata` is retained only by the development harness for provenance/split bookkeeping and is never exposed to route features. Legacy `metadata` is treated as eval-only provenance. Baseline scorers are `numeric` (including fractions, decimals, and percentages), -`json`, `one_of`, byte-sensitive `exact`, and `casefold_text`. `auto` tries -numeric and JSON before normalized prose text; code families automatically use -`exact` unless a dataset selects a future executable scorer. Unknown scorer names -are rejected. These transparent local scorers are development signals; they are -not presented as substitutes for the official evaluator. +type-aware `json`, `one_of`, byte-sensitive `exact`, and `casefold_text`. `auto` +tries numeric values, treats object targets as JSON and array targets as allowed +alternatives, then uses normalized prose text. Unknown scorer names and +non-array `one_of` targets are rejected. Code debugging/generation +cases must now declare a scorer explicitly: byte-exact scoring is valid only +when the task itself has one exact output; `one_of` can represent a bounded set +of accepted answers. Do not use either as evidence of general functional code +equivalence. These transparent local scorers produce proxy accuracy only; they +are not substitutes for the official LLM judge, especially for factual, +sentiment-reason, summary, NER, logic-explanation, and code semantics. + +Each file must contain one uniform `eval_metadata.split`. Observation files bind +the exact ordered dataset, expected task count, source implementation, runtime +configuration, endpoint mode, and scorer policy. When aggregating, pass one +`--require-route` for every intended candidate so a completely absent route is +also rejected. diff --git a/eval/datasets/calibration-v1.json b/eval/datasets/calibration-v1.json new file mode 100644 index 0000000..4e6bb83 --- /dev/null +++ b/eval/datasets/calibration-v1.json @@ -0,0 +1,363 @@ +[ + { + "id": "cal-factual-01", + "kind": "factual_qa", + "prompt": "Name the largest ocean by surface area. Answer with only its name.", + "expected_answer": "Pacific Ocean", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-factual-02", + "kind": "factual_qa", + "prompt": "Which chemical element has the symbol W? Answer with the element name only.", + "expected_answer": "Tungsten", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-factual-03", + "kind": "factual_qa", + "prompt": "Who wrote the novel The Left Hand of Darkness? Answer with the author's name only.", + "expected_answer": "Ursula K. Le Guin", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-factual-04", + "kind": "factual_qa", + "prompt": "What is the capital city of Slovenia? Answer with the city name only.", + "expected_answer": "Ljubljana", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-factual-05", + "kind": "factual_qa", + "prompt": "Name the process by which green plants convert light energy into chemical energy. Answer with the process name only.", + "expected_answer": "Photosynthesis", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-math-01", + "kind": "arithmetic", + "prompt": "Calculate 17 * 23. Return only the number.", + "expected_answer": "391", + "scorer": "numeric", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-math-02", + "kind": "arithmetic", + "prompt": "What is 15% of 260? Return only the number.", + "expected_answer": "39", + "scorer": "numeric", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-math-03", + "kind": "arithmetic", + "prompt": "Evaluate (84 / 7) + 19. Return only the number.", + "expected_answer": "31", + "scorer": "numeric", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-math-04", + "kind": "arithmetic", + "prompt": "Compute 5/6 - 1/4. Return the result as a reduced fraction only.", + "expected_answer": "7/12", + "scorer": "numeric", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-math-05", + "kind": "arithmetic", + "prompt": "Calculate how many jars remain when 8 cartons hold 14 jars each and 9 jars are removed. Return only the number.", + "expected_answer": "103", + "scorer": "numeric", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-sentiment-01", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: The update fixed every issue and made the app noticeably faster.", + "expected_answer": "Positive", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-sentiment-02", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: The package arrived cracked, and support never replied.", + "expected_answer": "Negative", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-sentiment-03", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nText: The meeting was moved from 2 p.m. to 3 p.m.", + "expected_answer": "Neutral", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-sentiment-04", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: The interface is elegant, but the program crashed twice during setup.", + "expected_answer": "Mixed", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-sentiment-05", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: I expected delays, yet the team delivered a day early.", + "expected_answer": "Positive", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-summary-01", + "kind": "summarisation", + "prompt": "Summarize the passage in exactly one sentence of no more than 12 words.\nPassage: MetroLink surveyed commuters about its timetable. Many requested earlier service, so the operator added two trains before 7 a.m.", + "expected_answer": [ + "MetroLink added two early trains after commuters requested earlier service.", + "Commuter feedback led MetroLink to add two trains before 7 a.m." + ], + "scorer": "one_of", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-summary-02", + "kind": "summarisation", + "prompt": "Summarize the passage as exactly two bullet points, each no more than eight words.\nPassage: The storage migration will happen on Saturday. During the work, files will be read-only for twenty minutes. Engineers expect no data loss.", + "expected_answer": "- Storage migration occurs Saturday.\n- Files remain read-only for twenty minutes.", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-summary-03", + "kind": "summarisation", + "prompt": "Provide a summary in exactly one sentence of no more than 15 words.\nArticle: Researchers monitored three shaded streets and three streets without trees during summer. The shaded streets were cooler and had lower measured particulate levels.", + "expected_answer": [ + "Tree-shaded streets were cooler and had lower particulate levels during summer.", + "Summer measurements linked tree-shaded streets with cooler temperatures and less particulate pollution." + ], + "scorer": "one_of", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-summary-04", + "kind": "summarisation", + "prompt": "Write a summary in exactly one sentence using no more than 10 words.\nText: The museum will waive admission fees every Wednesday evening beginning in May. The program is funded by a local arts grant.", + "expected_answer": [ + "An arts grant funds free museum admission on Wednesday evenings.", + "Museum admission becomes free Wednesday evenings starting in May." + ], + "scorer": "one_of", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-summary-05", + "kind": "summarisation", + "prompt": "List the key takeaways as exactly three bullet points, with at most six words per bullet.\nDocument: The team selected cedar for the outdoor benches. Assembly begins Monday. Delivery to the park is scheduled for Friday.", + "expected_answer": "- Cedar was selected.\n- Assembly begins Monday.\n- Delivery is scheduled Friday.", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-ner-01", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: On 14 March 2025, Maya Chen joined Northstar Robotics in Bristol.", + "expected_answer": {"person": ["Maya Chen"], "organization": ["Northstar Robotics"], "location": ["Bristol"], "date": ["14 March 2025"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-ner-02", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: UNESCO opened the exhibit in Dakar on Monday.", + "expected_answer": {"person": [], "organization": ["UNESCO"], "location": ["Dakar"], "date": ["Monday"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-ner-03", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: Amina Yusuf and Daniel Ortiz represented Aurora Hospitality Group at a conference in Quito.", + "expected_answer": {"person": ["Amina Yusuf", "Daniel Ortiz"], "organization": ["Aurora Hospitality Group"], "location": ["Quito"], "date": []}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-ner-04", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: Lakeview Press announced on 8 July that Priya Nair would lead its Toronto office.", + "expected_answer": {"person": ["Priya Nair"], "organization": ["Lakeview Press"], "location": ["Toronto"], "date": ["8 July"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-ner-05", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: Yesterday, Orion Labs sent Elena Petrova from Sofia to the Alpine Research Center.", + "expected_answer": {"person": ["Elena Petrova"], "organization": ["Orion Labs", "Alpine Research Center"], "location": ["Sofia"], "date": ["Yesterday"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-logic-01", + "kind": "logic", + "prompt": "Mara is older than Niko. Niko is older than Ivo. Who is the youngest? Return the name only.", + "expected_answer": "Ivo", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-logic-02", + "kind": "logic", + "prompt": "Logic puzzle: Every brass object conducts electricity. The ring is brass. Which conclusion must be true? Return only the conclusion: The ring conducts electricity, or The ring is magnetic.", + "expected_answer": "The ring conducts electricity", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-logic-03", + "kind": "logic", + "prompt": "Logic constraint: Exactly one of switches A and B is on. Switch A is off. Which switch must be on? Return A or B only.", + "expected_answer": "B", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-logic-04", + "kind": "logic", + "prompt": "Ordering constraints: Red is before Blue, and Green is after Blue. Which color must be last? Return the color only.", + "expected_answer": "Green", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-logic-05", + "kind": "logic", + "prompt": "Apply formal logic: If P is true, then Q is true. Q is false. What must be true about P? Return either 'P is true' or 'P is false' only.", + "expected_answer": "P is false", + "scorer": "casefold_text", + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-debug-01", + "kind": "code_debug", + "prompt": "Fix this Python function so it returns the last item, or None for an empty list. Preserve the signature and variable names. Return corrected code only. Do not use Markdown fences.\n```python\ndef last_or_none(items):\n if not items:\n return None\n return items[0]\n```", + "expected_answer": ["def last_or_none(items):\n if not items:\n return None\n return items[-1]"], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-debug-02", + "kind": "code_debug", + "prompt": "Fix the off-by-one bug in this JavaScript function. Preserve the signature and variable names. Return corrected code only. Do not use Markdown fences.\n```javascript\nfunction total(items) {\n let sum = 0;\n for (let i = 0; i <= items.length; i++) {\n sum += items[i];\n }\n return sum;\n}\n```", + "expected_answer": ["function total(items) {\n let sum = 0;\n for (let i = 0; i < items.length; i++) {\n sum += items[i];\n }\n return sum;\n}"], + "scorer": "one_of", + "task_metadata": {"language": "javascript"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-debug-03", + "kind": "code_debug", + "prompt": "Repair this SQL query so it selects invoices whose paid_at value is null. Return corrected code only. Do not use Markdown fences.\n```sql\nSELECT id FROM invoices WHERE paid_at = NULL;\n```", + "expected_answer": ["SELECT id FROM invoices WHERE paid_at IS NULL;"], + "scorer": "one_of", + "task_metadata": {"language": "sql"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-debug-04", + "kind": "code_debug", + "prompt": "Fix this Python countdown so it also prints zero. Preserve the signature and variable names. Return corrected code only. Do not use Markdown fences.\n```python\ndef countdown(n):\n while n > 0:\n print(n)\n n -= 1\n```", + "expected_answer": ["def countdown(n):\n while n >= 0:\n print(n)\n n -= 1"], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-debug-05", + "kind": "code_debug", + "prompt": "Fix this Java method so it compares the contents of two non-null strings. Return corrected code only. Do not use Markdown fences.\n```java\nstatic boolean same(String a, String b) {\n return a == b;\n}\n```", + "expected_answer": ["static boolean same(String a, String b) {\n return a.equals(b);\n}"], + "scorer": "one_of", + "task_metadata": {"language": "java"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-codegen-01", + "kind": "code_generation", + "prompt": "Write a Python function named is_even that accepts n and returns whether n is divisible by 2. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "def is_even(n):\n return n % 2 == 0", + "def is_even(n):\n return (n % 2) == 0" + ], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-codegen-02", + "kind": "code_generation", + "prompt": "Write a JavaScript function named clamp that returns x limited to the inclusive range min through max. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "function clamp(x, min, max) {\n return Math.min(max, Math.max(min, x));\n}", + "const clamp = (x, min, max) => Math.min(max, Math.max(min, x));" + ], + "scorer": "one_of", + "task_metadata": {"language": "javascript"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-codegen-03", + "kind": "code_generation", + "prompt": "Write a SQL query that returns department_id and the average salary as avg_salary from employees, grouped by department_id. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "SELECT department_id, AVG(salary) AS avg_salary FROM employees GROUP BY department_id;", + "SELECT department_id, AVG(salary) AS avg_salary\nFROM employees\nGROUP BY department_id;" + ], + "scorer": "one_of", + "task_metadata": {"language": "sql"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-codegen-04", + "kind": "code_generation", + "prompt": "Write a Bash function named make_backup that copies the file given as its first argument to the same path with .bak appended. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "make_backup() {\n cp -- \"$1\" \"$1.bak\"\n}", + "function make_backup() {\n cp -- \"$1\" \"$1.bak\"\n}" + ], + "scorer": "one_of", + "task_metadata": {"language": "bash"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + }, + { + "id": "cal-codegen-05", + "kind": "code_generation", + "prompt": "Write a Python class Counter whose constructor sets value to zero and whose increment method adds one and returns the new value. Return code only. Do not use Markdown fences.", + "expected_answer": ["class Counter:\n def __init__(self):\n self.value = 0\n\n def increment(self):\n self.value += 1\n return self.value"], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "calibration", "provenance": "hand-authored-original-v1"} + } +] diff --git a/eval/datasets/holdout-v1.json b/eval/datasets/holdout-v1.json new file mode 100644 index 0000000..8ed978e --- /dev/null +++ b/eval/datasets/holdout-v1.json @@ -0,0 +1,369 @@ +[ + { + "id": "hold-factual-01", + "kind": "factual_qa", + "prompt": "What is the capital city of Mongolia? Answer with the city name only.", + "expected_answer": "Ulaanbaatar", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-factual-02", + "kind": "factual_qa", + "prompt": "Which scientist formulated the three laws of planetary motion? Answer with the scientist's name only.", + "expected_answer": "Johannes Kepler", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-factual-03", + "kind": "factual_qa", + "prompt": "Name the world's deepest freshwater lake. Answer with the lake name only.", + "expected_answer": "Lake Baikal", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-factual-04", + "kind": "factual_qa", + "prompt": "What is the official currency of South Korea? Answer with the currency name only.", + "expected_answer": "South Korean won", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-factual-05", + "kind": "factual_qa", + "prompt": "In what year did the Berlin Wall fall? Return the four-digit year only.", + "expected_answer": "1989", + "scorer": "numeric", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-math-01", + "kind": "arithmetic", + "prompt": "Calculate 29 * 16. Return only the number.", + "expected_answer": "464", + "scorer": "numeric", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-math-02", + "kind": "arithmetic", + "prompt": "What is 12.5% of 640? Return only the number.", + "expected_answer": "80", + "scorer": "numeric", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-math-03", + "kind": "arithmetic", + "prompt": "Evaluate (144 / 12) ** 2. Return only the number.", + "expected_answer": "144", + "scorer": "numeric", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-math-04", + "kind": "arithmetic", + "prompt": "Compute 7/9 + 5/12. Return the result as a reduced fraction only.", + "expected_answer": "43/36", + "scorer": "numeric", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-math-05", + "kind": "arithmetic", + "prompt": "Six buses each have 48 seats. If 37 seats are empty in total, how many seats are occupied? Return only the number.", + "expected_answer": "251", + "scorer": "numeric", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-sentiment-01", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: Setup took seconds, and every feature worked exactly as promised.", + "expected_answer": "Positive", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-sentiment-02", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: The battery died after an hour and the replacement was also defective.", + "expected_answer": "Negative", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-sentiment-03", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nText: The parcel weighs 2.4 kilograms and contains three manuals.", + "expected_answer": "Neutral", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-sentiment-04", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: The camera takes beautiful photos, although its menus are frustratingly slow.", + "expected_answer": "Mixed", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-sentiment-05", + "kind": "sentiment", + "prompt": "Classify the sentiment as Positive, Negative, Neutral, or Mixed. Return one label only.\nReview: It is neither impressive nor disappointing; it simply does the stated job.", + "expected_answer": "Neutral", + "scorer": "casefold_text", + "task_metadata": {"allowed_labels": ["Positive", "Negative", "Neutral", "Mixed"]}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-summary-01", + "kind": "summarisation", + "prompt": "Summarize the passage in exactly one sentence of no more than 12 words.\nPassage: Harbor City installed sensors in public bins. Collection crews now visit only bins that are nearly full, reducing unnecessary trips.", + "expected_answer": [ + "Bin sensors help Harbor City reduce unnecessary collection trips.", + "Harbor City uses bin sensors to avoid unnecessary collection trips." + ], + "scorer": "one_of", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-summary-02", + "kind": "summarisation", + "prompt": "Summarize the passage as exactly two bullet points, each no more than eight words.\nPassage: Registration closes Thursday at noon. Late applications will not be accepted. Accepted applicants will receive email confirmation on Monday.", + "expected_answer": "- Registration closes Thursday at noon.\n- Accepted applicants receive confirmation Monday.", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-summary-03", + "kind": "summarisation", + "prompt": "Provide a summary in exactly one sentence of no more than 15 words.\nArticle: A neighborhood bakery replaced disposable cups with a deposit-return system. In its first month, customers returned 92 percent of the reusable cups.", + "expected_answer": [ + "The bakery's deposit system achieved a 92 percent reusable-cup return rate.", + "Customers returned 92 percent of reusable cups under the bakery's deposit system." + ], + "scorer": "one_of", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-summary-04", + "kind": "summarisation", + "prompt": "Write a summary in exactly one sentence using no more than 10 words.\nText: The north trail remains closed after heavy rain damaged a footbridge. Rangers expect repairs to finish by the end of June.", + "expected_answer": [ + "Bridge damage closes the north trail until late June.", + "The north trail should reopen after bridge repairs in June." + ], + "scorer": "one_of", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-summary-05", + "kind": "summarisation", + "prompt": "List the key takeaways as exactly three bullet points, with at most six words per bullet.\nDocument: The council approved the playground design. Construction starts in August. The playground is expected to open in October.", + "expected_answer": "- Playground design was approved.\n- Construction starts in August.\n- Opening is expected in October.", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-ner-01", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: On 3 November, Rafael Costa presented the award for Meridian Energy in Lisbon.", + "expected_answer": {"person": ["Rafael Costa"], "organization": ["Meridian Energy"], "location": ["Lisbon"], "date": ["3 November"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-ner-02", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: Tomorrow, Blue Finch Theatre will begin rehearsals in Osaka.", + "expected_answer": {"person": [], "organization": ["Blue Finch Theatre"], "location": ["Osaka"], "date": ["Tomorrow"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-ner-03", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: Laila Haddad and Owen Price visited Cedar Point Observatory near Flagstaff.", + "expected_answer": {"person": ["Laila Haddad", "Owen Price"], "organization": ["Cedar Point Observatory"], "location": ["Flagstaff"], "date": []}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-ner-04", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: Solstice Bank appointed Eric Mensah to manage its Accra branch on 1 February 2026.", + "expected_answer": {"person": ["Eric Mensah"], "organization": ["Solstice Bank"], "location": ["Accra"], "date": ["1 February 2026"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-ner-05", + "kind": "entity_extraction", + "prompt": "Extract all person, organization, location, and date entities from the text. Return one JSON object with exactly those four keys and arrays of exact source spans; use empty arrays when needed.\nText: Nova Health and Eastgate University will host Samira Bose in Nairobi next Tuesday.", + "expected_answer": {"person": ["Samira Bose"], "organization": ["Nova Health", "Eastgate University"], "location": ["Nairobi"], "date": ["next Tuesday"]}, + "scorer": "json", + "task_metadata": {"output_format": "json"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-logic-01", + "kind": "logic", + "prompt": "Lina is faster than Omar. Omar is faster than Priya. Who is the fastest? Return the name only.", + "expected_answer": "Lina", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-logic-02", + "kind": "logic", + "prompt": "Logic constraint: All poets are readers. No readers are robots. Can a poet be a robot? Return Yes or No only.", + "expected_answer": "No", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-logic-03", + "kind": "logic", + "prompt": "Logic constraint: Exactly one of boxes A, B, and C contains a ruby. B does not contain it, and C does not contain it. Which box must contain the ruby? Return A, B, or C only.", + "expected_answer": "A", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-logic-04", + "kind": "logic", + "prompt": "Ordering constraints: Task A occurs before Task C, and Task B occurs after Task C. Which task must be last? Return A, B, or C only.", + "expected_answer": "B", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-logic-05", + "kind": "logic", + "prompt": "Logic constraint: Exactly one route is taken: north or east. The north route is not taken. Which route must be taken? Return north or east only.", + "expected_answer": "east", + "scorer": "casefold_text", + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-debug-01", + "kind": "code_debug", + "prompt": "Fix this Python function so it returns zero for an empty list and otherwise returns the arithmetic mean. Preserve the signature and variable names. Return corrected code only. Do not use Markdown fences.\n```python\ndef mean(values):\n return sum(values) / len(values)\n```", + "expected_answer": ["def mean(values):\n if not values:\n return 0\n return sum(values) / len(values)"], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-debug-02", + "kind": "code_debug", + "prompt": "Fix this JavaScript function so it returns the doubled numbers instead of an array of undefined values. Preserve the signature and variable names. Return corrected code only. Do not use Markdown fences.\n```javascript\nfunction doubled(numbers) {\n return numbers.map(n => { n * 2; });\n}\n```", + "expected_answer": [ + "function doubled(numbers) {\n return numbers.map(n => n * 2);\n}", + "function doubled(numbers) {\n return numbers.map(n => { return n * 2; });\n}" + ], + "scorer": "one_of", + "task_metadata": {"language": "javascript"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-debug-03", + "kind": "code_debug", + "prompt": "Repair this SQL query so it returns one row per customer with the number of orders. Return corrected code only. Do not use Markdown fences.\n```sql\nSELECT customer_id, COUNT(*) FROM orders;\n```", + "expected_answer": ["SELECT customer_id, COUNT(*) FROM orders GROUP BY customer_id;"], + "scorer": "one_of", + "task_metadata": {"language": "sql"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-debug-04", + "kind": "code_debug", + "prompt": "Fix this Python function so it removes and returns the final list item rather than the first. Preserve the signature and variable names. Return corrected code only. Do not use Markdown fences.\n```python\ndef take_last(items):\n return items.pop(0)\n```", + "expected_answer": ["def take_last(items):\n return items.pop()", "def take_last(items):\n return items.pop(-1)"], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-debug-05", + "kind": "code_debug", + "prompt": "Fix this C function so it allocates space for ten integers rather than ten bytes. Return corrected code only. Do not use Markdown fences.\n```c\nint *make_buffer(void) {\n return malloc(10);\n}\n```", + "expected_answer": [ + "int *make_buffer(void) {\n return malloc(10 * sizeof(int));\n}", + "int *make_buffer(void) {\n return malloc(sizeof(int) * 10);\n}" + ], + "scorer": "one_of", + "task_metadata": {"language": "c"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-codegen-01", + "kind": "code_generation", + "prompt": "Write a Python function named square_all that accepts values and returns a new list containing each value squared. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "def square_all(values):\n return [value ** 2 for value in values]", + "def square_all(values):\n return [value * value for value in values]" + ], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-codegen-02", + "kind": "code_generation", + "prompt": "Write a JavaScript function named firstOrNull that returns the first array element or null when the array is empty. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "function firstOrNull(items) {\n return items.length ? items[0] : null;\n}", + "const firstOrNull = items => items.length ? items[0] : null;" + ], + "scorer": "one_of", + "task_metadata": {"language": "javascript"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-codegen-03", + "kind": "code_generation", + "prompt": "Write a SQL query that returns category and the maximum price as max_price from products, grouped by category. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "SELECT category, MAX(price) AS max_price FROM products GROUP BY category;", + "SELECT category, MAX(price) AS max_price\nFROM products\nGROUP BY category;" + ], + "scorer": "one_of", + "task_metadata": {"language": "sql"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-codegen-04", + "kind": "code_generation", + "prompt": "Write a Bash function named line_count that prints the number of lines in the file given as its first argument. Return code only. Do not use Markdown fences.", + "expected_answer": [ + "line_count() {\n wc -l < \"$1\"\n}", + "function line_count() {\n wc -l < \"$1\"\n}" + ], + "scorer": "one_of", + "task_metadata": {"language": "bash"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + }, + { + "id": "hold-codegen-05", + "kind": "code_generation", + "prompt": "Write a Python class Toggle whose constructor sets enabled to False and whose flip method toggles enabled and returns the new value. Return code only. Do not use Markdown fences.", + "expected_answer": ["class Toggle:\n def __init__(self):\n self.enabled = False\n\n def flip(self):\n self.enabled = not self.enabled\n return self.enabled"], + "scorer": "one_of", + "task_metadata": {"language": "python"}, + "eval_metadata": {"split": "holdout", "provenance": "hand-authored-original-v1"} + } +] diff --git a/eval/evaluate_policy.py b/eval/evaluate_policy.py new file mode 100644 index 0000000..c2537b8 --- /dev/null +++ b/eval/evaluate_policy.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Replay a measured route table over route observations. + +This composes independent full-coverage route runs into the same first-success +fallthrough order used by the scored runtime. It reports transparent local proxy +accuracy and total Fireworks tokens, including failed attempts before the +selected route and conservative task-deadline composition. +""" + +from __future__ import annotations + +import argparse +import json +import math +from collections import Counter, defaultdict +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from eval.compare_routes import aggregate +from fallthrough.policy.route_table import RouteTable +from fallthrough.tasks.kinds import TaskKind + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("observations", nargs="+", type=Path) + parser.add_argument("--route-table", type=Path, required=True) + parser.add_argument("--task-timeout", type=float, default=29.0) + parser.add_argument("--output", type=Path) + return parser.parse_args(argv) + + +def evaluate_policy( + paths: list[Path], + route_table: RouteTable, + *, + task_timeout_seconds: float = 29.0, +) -> dict[str, Any]: + if not math.isfinite(task_timeout_seconds) or not 0 < task_timeout_seconds < 30: + raise ValueError("task_timeout_seconds must be positive and below 30") + required_routes = {name for kind in TaskKind for name in route_table.for_kind(kind)} + # Reuse the strict comparator audit before composing rows: one dataset, + # complete task coverage, reproducible correctness, and coherent run IDs. + aggregate(paths, required_routes=required_routes) + + by_task: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict) + order: list[str] = [] + dataset_id: str | None = None + dataset_split: str | None = None + for path in paths: + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + features = row.get("route_features") + if isinstance(features, dict): + configured = features.get("configured_model") + served = features.get("served_model") + if ( + isinstance(configured, str) + and configured + and isinstance(served, str) + and served + and configured != served + ): + raise ValueError( + "model identity mismatch must be verified before policy promotion: " + f"requested={configured!r}, reported={served!r}" + ) + task_id = str(row["task_id"]) + route_name = str(row["route_name"]) + if task_id not in by_task: + order.append(task_id) + by_task[task_id][route_name] = row + dataset_id = str(row["dataset_fingerprint"]) + dataset_split = str(row["dataset_split"]) + + correct_count = 0 + failed_count = 0 + all_usage_known = True + total_observed_tokens = 0 + selected_routes: Counter[str] = Counter() + family_rows: dict[str, list[dict[str, Any]]] = defaultdict(list) + task_results: list[dict[str, Any]] = [] + + for task_id in order: + rows = by_task[task_id] + first = next(iter(rows.values())) + kind = _policy_kind(rows, first) + route_names = route_table.for_kind(kind) + selected: dict[str, Any] | None = None + attempted: list[str] = [] + task_tokens = 0 + task_usage_known = True + task_latency_ms = 0.0 + deadline_exceeded = False + for route_name in route_names: + row = rows[route_name] + attempted.append(route_name) + route_latency_ms = float(row["latency_ms"]) + remaining_ms = task_timeout_seconds * 1000.0 - task_latency_ms + if route_latency_ms >= remaining_ms: + deadline_exceeded = True + task_latency_ms = task_timeout_seconds * 1000.0 + if route_name.startswith("fw_"): + task_usage_known = False + break + task_latency_ms += route_latency_ms + task_tokens += int(row["prompt_tokens"]) + int(row["completion_tokens"]) + task_usage_known = task_usage_known and bool(row["usage_known"]) + if not bool(row["failed"]): + selected = row + break + + correct = bool(selected is not None and selected["correct"]) + correct_count += int(correct) + failed_count += int(selected is None) + total_observed_tokens += task_tokens + all_usage_known = all_usage_known and task_usage_known + selected_name = str(selected["route_name"]) if selected is not None else None + if selected_name is not None: + selected_routes[selected_name] += 1 + family = str(first.get("task_family") or kind.value) + composed = { + "task_id": task_id, + "task_family": family, + "routing_kind": kind.value, + "attempted_routes": attempted, + "selected_route": selected_name, + "correct": correct, + "failed": selected is None, + "deadline_exceeded": deadline_exceeded, + "composed_latency_ms": task_latency_ms, + "fireworks_tokens": task_tokens if task_usage_known else None, + "observed_minimum_fireworks_tokens": task_tokens, + "usage_known": task_usage_known, + } + family_rows[family].append(composed) + task_results.append(composed) + + count = len(task_results) + route_table_payload = route_table.canonical_payload() + return { + "dataset_fingerprint": dataset_id, + "dataset_split": dataset_split, + "route_table": route_table_payload, + "route_table_sha256": route_table.canonical_sha256(), + "route_table_source": route_table.source, + "task_count": count, + "correct_count": correct_count, + "proxy_accuracy": correct_count / count, + "accuracy": correct_count / count, + "accuracy_semantics": "transparent_local_proxy_not_official_llm_judge", + "task_timeout_seconds": task_timeout_seconds, + "deadline_simulation": "conservative_sum_of_observed_route_wall_latency", + "total_fireworks_tokens": total_observed_tokens if all_usage_known else None, + "total_observed_minimum_fireworks_tokens": total_observed_tokens, + "token_metrics_complete": all_usage_known, + "failure_rate": failed_count / count, + "selected_route_counts": dict(sorted(selected_routes.items())), + "families": {family: _family_metrics(rows) for family, rows in sorted(family_rows.items())}, + "tasks": task_results, + } + + +def _policy_kind(rows: dict[str, dict[str, Any]], first: dict[str, Any]) -> TaskKind: + classified = TaskKind(str(first["classified_task_kind"])) + deterministic = rows.get("deterministic") + if deterministic is None: + return classified + # The benchmark's deterministic route records its objective-kind override; + # production applies that same override before consulting the route table. + return TaskKind(str(deterministic["task_kind"])) + + +def _family_metrics(rows: list[dict[str, Any]]) -> dict[str, Any]: + complete = all(bool(row["usage_known"]) for row in rows) + observed = sum(int(row["observed_minimum_fireworks_tokens"]) for row in rows) + return { + "task_count": len(rows), + "proxy_accuracy": sum(bool(row["correct"]) for row in rows) / len(rows), + "accuracy": sum(bool(row["correct"]) for row in rows) / len(rows), + "total_fireworks_tokens": observed if complete else None, + "total_observed_minimum_fireworks_tokens": observed, + "token_metrics_complete": complete, + "failure_rate": sum(bool(row["failed"]) for row in rows) / len(rows), + } + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + route_table = RouteTable.from_json(args.route_table) + result = evaluate_policy( + args.observations, + route_table, + task_timeout_seconds=args.task_timeout, + ) + text = json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if args.output is None: + print(text, end="") + else: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/report.py b/eval/report.py index 72c51dd..d7a1987 100644 --- a/eval/report.py +++ b/eval/report.py @@ -15,20 +15,29 @@ def main() -> int: args = parser.parse_args() payload = json.loads(args.aggregate.read_text(encoding="utf-8")) lines = [ - "| Task kind | Route | Cases | Accuracy | Mean FW tokens | " - "Unknown usage | Invalid output | Failure rate |", - "|---|---|---:|---:|---:|---:|---:|---:|", + "| Task kind | Route | Cases | Proxy accuracy | Total FW tokens | Mean FW tokens | " + "Token evidence complete | Unknown usage | Invalid output | Failure rate | " + "Startup s | Peak MiB |", + "|---|---|---:|---:|---:|---:|---|---:|---:|---:|---:|---:|", ] for kind, routes in sorted(payload.items()): for route, metrics in sorted(routes.items()): mean_tokens = metrics["mean_fireworks_tokens"] token_text = f"{mean_tokens:.1f}" if mean_tokens is not None else "unknown" + total_tokens = metrics.get("total_fireworks_tokens") + total_text = str(total_tokens) if total_tokens is not None else "unknown" invalid_rate = metrics.get("invalid_output_rate") invalid_text = f"{invalid_rate:.3f}" if invalid_rate is not None else "n/a" + startup = metrics.get("startup_time_seconds") + startup_text = f"{startup:.3f}" if startup is not None else "n/a" + peak_memory = metrics.get("peak_memory_bytes") + peak_text = f"{peak_memory / (1024**2):.1f}" if peak_memory is not None else "n/a" + token_complete = "yes" if metrics.get("token_metrics_complete") else "no" lines.append( f"| {kind} | {route} | {metrics['count']} | {metrics['accuracy']:.3f} | " - f"{token_text} | {metrics['usage_unknown_rate']:.3f} | {invalid_text} | " - f"{metrics['failure_rate']:.3f} |" + f"{total_text} | {token_text} | {token_complete} | " + f"{metrics['usage_unknown_rate']:.3f} | " + f"{invalid_text} | {metrics['failure_rate']:.3f} | {startup_text} | {peak_text} |" ) text = "\n".join(lines) + "\n" if args.output: diff --git a/eval/run_routes.py b/eval/run_routes.py index 4166d7f..3ff8bf8 100644 --- a/eval/run_routes.py +++ b/eval/run_routes.py @@ -5,18 +5,32 @@ import argparse import asyncio +import hashlib import json import math import os import tempfile import time -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from pathlib import Path +from urllib.parse import urlsplit -from eval.common import EvalCase, load_cases, score_answer +from eval.common import ( + SCORING_POLICY, + EvalCase, + benchmark_context_fingerprint, + dataset_fingerprint, + dataset_split, + load_cases, + observation_correct, + resolve_scorer, +) +from fallthrough import __version__ from fallthrough.config import Settings from fallthrough.fireworks.client import FireworksClient -from fallthrough.routes.base import Route +from fallthrough.inference import DEFAULT_MAX_TOKENS_BY_KIND, PROMPT_POLICY +from fallthrough.local import source_tree_fingerprint +from fallthrough.routes.base import Route, RouteResult from fallthrough.routes.deterministic import DeterministicRoute from fallthrough.routes.fireworks_gemma import FireworksGemmaRoute from fallthrough.routes.fireworks_model import FireworksModelRoute, fireworks_route_name @@ -60,10 +74,14 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def _requested_routes(values: list[str] | None) -> set[str]: +def _requested_routes( + values: list[str] | None, +) -> set[str]: requested = set(values or ["all"]) if "all" in requested: - return {"deterministic", "fw_gemma", "fw_generic"} + # `fw_generic` covers every exact ALLOWED_MODELS entry, including + # Gemma. Avoid a duplicate call through the late-bound fw_gemma alias. + return {"deterministic", "fw_generic"} return requested @@ -88,7 +106,7 @@ def _build_routes( client = FireworksClient( settings.fireworks_base_url, settings.fireworks_api_key, - settings.allowed_models if settings.allowed_models_configured else None, + settings.allowed_models, connect_timeout_s=settings.connect_timeout_seconds, read_timeout_s=settings.request_timeout_seconds, max_retries=settings.transport_retries, @@ -111,11 +129,7 @@ def _build_routes( if not benchmark_models: raise ValueError("fw_generic requires --model or ALLOWED_MODELS") for model in benchmark_models: - if model == settings.gemma_model: - if "fw_gemma" not in requested: - routes.append( - FireworksGemmaRoute(client, model, format_retries=settings.format_retries) - ) + if model == settings.gemma_model and "fw_gemma" in requested: continue routes.append( FireworksModelRoute( @@ -133,71 +147,173 @@ async def run_benchmark( routes: list[Route], *, concurrency: int, - timeout_seconds: float = 30.0, + timeout_seconds: float = 25.0, + run_context: Mapping[str, object] | None = None, ) -> list[RouteObservation]: if concurrency <= 0: raise ValueError("concurrency must be positive") if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: raise ValueError("timeout_seconds must be positive") + if timeout_seconds >= 30.0: + raise ValueError("timeout_seconds must be below the final grader's 30-second limit") + # Reject non-auditable scorer contracts before any route can spend tokens. + for case in cases: + preflight_features = extract_features(case.task) + preflight_kind = classify_task(case.task, preflight_features) + resolve_scorer(case, preflight_kind) semaphore = asyncio.Semaphore(concurrency) + dataset_id = dataset_fingerprint(cases) + split = dataset_split(cases) + dataset_count = len(cases) + common_source = Path(__file__).with_name("common.py") + shared_implementation_id = source_tree_fingerprint(common_source) + benchmark_context_id = benchmark_context_fingerprint( + dataset_id=dataset_id, + dataset_split_name=split, + dataset_task_count=dataset_count, + shared_implementation_sha256=shared_implementation_id, + ) + implementation_id = source_tree_fingerprint( + Path(__file__), + common_source, + ) + evidence_context = dict(run_context or {}) + route_evidence: dict[int, tuple[str, dict[str, object]]] = {} + for route in routes: + route_evidence[id(route)] = _route_run_fingerprint( + route, + dataset_id, + timeout_seconds, + concurrency, + implementation_id, + shared_implementation_id, + benchmark_context_id, + evidence_context, + ) - async def evaluate(case: EvalCase, route: Route) -> RouteObservation | None: - features = extract_features(case.task) - kind = case.kind or classify_task(case.task, features) - objective_kind = route.objective_kind(case.task) - if objective_kind is not None: - kind = objective_kind - if not route.supports(kind): - return None + async def evaluate(case: EvalCase, route: Route) -> RouteObservation: + run_id, route_configuration = route_evidence[id(route)] model = getattr(route, "model", "") started = time.perf_counter() + classified_kind = TaskKind.UNKNOWN + kind = TaskKind.UNKNOWN + prompt_length = len(case.task.prompt) + numeric_ratio = 0.0 + has_code = False + has_json = False + route_invoked = False + queue_latency_ms = 0.0 try: - async with semaphore: - result = await asyncio.wait_for( - route.execute(case.task, kind), - timeout=timeout_seconds, + features = extract_features(case.task) + prompt_length = features.prompt_length + numeric_ratio = features.numeric_ratio + has_code = features.has_code + has_json = features.has_json + classified_kind = classify_task(case.task, features) + kind = classified_kind + objective_kind = route.objective_kind(case.task) + if objective_kind is not None: + kind = objective_kind + if not route.supports(kind): + return _failed_observation( + case, + route, + kind, + classified_kind, + dataset_id, + split, + dataset_count, + implementation_id, + shared_implementation_id, + benchmark_context_id, + route_configuration, + run_id, + model, + prompt_length, + numeric_ratio, + has_code, + has_json, + "unsupported_kind", + (time.perf_counter() - started) * 1000.0, + usage_known=True, ) + + async def execute() -> RouteResult: + nonlocal queue_latency_ms, route_invoked + async with semaphore: + queue_latency_ms = (time.perf_counter() - started) * 1000.0 + route_invoked = True + return await asyncio.wait_for( + route.execute(case.task, kind), + timeout=timeout_seconds, + ) + + result = await execute() except asyncio.TimeoutError: return _failed_observation( case, route, kind, + classified_kind, + dataset_id, + split, + dataset_count, + implementation_id, + shared_implementation_id, + benchmark_context_id, + route_configuration, + run_id, model, - features.prompt_length, - features.numeric_ratio, - features.has_code, - features.has_json, + prompt_length, + numeric_ratio, + has_code, + has_json, "route_timeout", (time.perf_counter() - started) * 1000.0, + usage_known=not route.name.startswith("fw_") or not route_invoked, ) except Exception as exc: return _failed_observation( case, route, kind, + classified_kind, + dataset_id, + split, + dataset_count, + implementation_id, + shared_implementation_id, + benchmark_context_id, + route_configuration, + run_id, model, - features.prompt_length, - features.numeric_ratio, - features.has_code, - features.has_json, + prompt_length, + numeric_ratio, + has_code, + has_json, f"route_exception:{type(exc).__name__}", (time.perf_counter() - started) * 1000.0, + usage_known=not route.name.startswith("fw_") or not route_invoked, ) validation = _validation_fields(result.metadata) - scorer = ( - "exact" - if case.scorer == "auto" and kind in {TaskKind.CODE_DEBUG, TaskKind.CODE_GENERATION} - else case.scorer - ) + scorer = resolve_scorer(case, classified_kind) + failed = not result.success + served_model = result.metadata.get("model") return RouteObservation( task_id=case.task.id, task_kind=kind, route_name=route.name, - correct=result.success and score_answer(result.answer, case.expected_answer, scorer), + correct=observation_correct( + result.answer, + case.expected_answer, + scorer, + failed=failed, + output_valid=validation[0], + ), prompt_tokens=result.fireworks_prompt_tokens, completion_tokens=result.fireworks_completion_tokens, - latency_ms=result.latency_ms, - failed=not result.success, + latency_ms=(time.perf_counter() - started) * 1000.0, + failed=failed, usage_known=result.fireworks_usage_known, output_valid=validation[0], validation_score=validation[1], @@ -207,22 +323,49 @@ async def evaluate(case: EvalCase, route: Route) -> RouteObservation | None: "numeric_ratio": features.numeric_ratio, "has_code": features.has_code, "has_json": features.has_json, - "model": model, + "configured_model": str(model), + "served_model": served_model if isinstance(served_model, str) else "", + "route_latency_ms": result.latency_ms, + "queue_latency_ms": queue_latency_ms, "validator_score": _validator_score(result.metadata), }, answer=result.answer, expected_answer=case.expected_answer, + declared_task_kind=case.kind.value if case.kind is not None else None, + classified_task_kind=classified_kind.value, + task_family=case.kind.value if case.kind is not None else classified_kind.value, + dataset_fingerprint=dataset_id, + dataset_split=split, + run_fingerprint=run_id, + dataset_task_count=dataset_count, + implementation_sha256=implementation_id, + shared_implementation_sha256=shared_implementation_id, + benchmark_context_fingerprint=benchmark_context_id, + declared_scorer=case.scorer, + effective_scorer=scorer, + scoring_policy=SCORING_POLICY, + run_configuration=route_configuration, ) - work = [evaluate(case, route) for case in cases for route in routes] - observations = await asyncio.gather(*work) - return [observation for observation in observations if observation is not None] + observations: list[RouteObservation] = [] + for route in routes: + observations.extend(await asyncio.gather(*(evaluate(case, route) for case in cases))) + return observations def _failed_observation( case: EvalCase, route: Route, kind: TaskKind, + classified_kind: TaskKind, + dataset_id: str, + split: str, + dataset_count: int, + implementation_id: str, + shared_implementation_id: str, + benchmark_context_id: str, + run_configuration: dict[str, object], + run_id: str, model: object, prompt_length: int, numeric_ratio: float, @@ -230,7 +373,10 @@ def _failed_observation( has_json: bool, reason: str, latency_ms: float, + *, + usage_known: bool, ) -> RouteObservation: + scorer = resolve_scorer(case, classified_kind) return RouteObservation( task_id=case.task.id, task_kind=kind, @@ -240,18 +386,108 @@ def _failed_observation( completion_tokens=0, latency_ms=latency_ms, failed=True, - usage_known=not route.name.startswith("fw_"), + usage_known=usage_known, route_features={ "prompt_length": prompt_length, "numeric_ratio": numeric_ratio, "has_code": has_code, "has_json": has_json, - "model": str(model), + "configured_model": str(model), + "served_model": "", "failure": reason, }, answer="", expected_answer=case.expected_answer, + declared_task_kind=case.kind.value if case.kind is not None else None, + classified_task_kind=classified_kind.value, + task_family=case.kind.value if case.kind is not None else classified_kind.value, + dataset_fingerprint=dataset_id, + dataset_split=split, + run_fingerprint=run_id, + dataset_task_count=dataset_count, + implementation_sha256=implementation_id, + shared_implementation_sha256=shared_implementation_id, + benchmark_context_fingerprint=benchmark_context_id, + declared_scorer=case.scorer, + effective_scorer=scorer, + scoring_policy=SCORING_POLICY, + run_configuration=run_configuration, + ) + + +def _route_run_fingerprint( + route: Route, + dataset_id: str, + timeout_seconds: float, + concurrency: int, + implementation_id: str, + shared_implementation_id: str, + benchmark_context_id: str, + run_context: Mapping[str, object], +) -> tuple[str, dict[str, object]]: + supported_kinds = getattr(route, "_supported_kinds", None) + configuration = { + "concurrency": concurrency, + "dataset_fingerprint": dataset_id, + "fallthrough_version": __version__, + "format_retries": getattr(route, "_format_retries", None), + "implementation_sha256": implementation_id, + "shared_implementation_sha256": shared_implementation_id, + "benchmark_context_fingerprint": benchmark_context_id, + "max_tokens": repr(getattr(route, "_max_tokens", None)), + "model": getattr(route, "model", None), + "prompt_policy": PROMPT_POLICY, + "prompt_suffix": getattr(route, "_prompt_suffix", None), + "prompt_suffix_policy": getattr(route, "_prompt_suffix_policy", None), + "route_class": type(route).__name__, + "route_name": route.name, + "reasoning_effort": getattr(route, "_reasoning_effort", None), + "samples": getattr(route, "_samples", None), + "supported_kinds": ( + sorted(kind.value for kind in supported_kinds) if supported_kinds is not None else None + ), + "temperature": getattr(route, "_temperature", None), + "timeout_seconds": timeout_seconds, + "token_budgets": DEFAULT_MAX_TOKENS_BY_KIND, + "run_context": dict(run_context), + } + encoded = json.dumps( + configuration, + ensure_ascii=True, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest(), configuration + + +def _run_context(settings: Settings, client: FireworksClient | None) -> dict[str, object]: + endpoint = client.endpoint_url if client is not None else None + endpoint_sha256 = ( + hashlib.sha256(endpoint.encode("utf-8")).hexdigest() if endpoint is not None else None ) + allowlist_payload = json.dumps( + sorted(settings.allowed_models), + ensure_ascii=True, + separators=(",", ":"), + ) + endpoint_mode = "offline" + if endpoint is not None: + hostname = urlsplit(endpoint).hostname + endpoint_mode = ( + "development_explicit" + if hostname and hostname.rstrip(".").casefold() == "api.fireworks.ai" + else "evaluator_injected" + ) + return { + "allowlist_sha256": hashlib.sha256(allowlist_payload.encode("utf-8")).hexdigest(), + "connect_timeout_seconds": settings.connect_timeout_seconds, + "endpoint_mode": endpoint_mode, + "endpoint_sha256": endpoint_sha256, + "format_retries": settings.format_retries, + "request_timeout_seconds": settings.request_timeout_seconds, + "transport_retries": settings.transport_retries, + } def _validator_score(metadata: dict[str, object]) -> float: @@ -310,9 +546,10 @@ async def async_main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) cases = load_cases(args.dataset) settings = Settings.from_env() + requested_routes = _requested_routes(args.route) routes, client = _build_routes( settings, - _requested_routes(args.route), + requested_routes, args.model, args.self_consistency, ) @@ -326,6 +563,7 @@ async def async_main(argv: Sequence[str] | None = None) -> int: if args.route_timeout is None else args.route_timeout ), + run_context=_run_context(settings, client), ) _atomic_write_jsonl(args.output, observations) finally: diff --git a/policy/README.md b/policy/README.md new file mode 100644 index 0000000..1aca503 --- /dev/null +++ b/policy/README.md @@ -0,0 +1,33 @@ +# Packaged measured-policy artifacts + +This directory contains the schema and, only after compliant measurement, the +release policy artifacts copied into the scored image. + +The repository intentionally contains no active `measured.json` or +`evidence.json`. The Docker build arguments default to empty, so ordinary builds +continue to use `baseline-unmeasured`. Producing and reviewing the evidence +sidecar from the final policy replay remains an explicit release step; this +mechanism performs no automatic policy promotion. + +Do not commit or package a `measured.json` or `evidence.json` merely to enable +the mechanism. A release policy must be generated from exact-model evidence and +must retain its exact `fw_model:` route names. + +When valid release artifacts exist, build the image with internal defaults that +point to both files: + +```bash +docker build --platform linux/amd64 \ + --build-arg FALLTHROUGH_PACKAGED_ROUTE_TABLE=/app/policy/measured.json \ + --build-arg FALLTHROUGH_PACKAGED_POLICY_EVIDENCE=/app/policy/evidence.json \ + --tag fallthrough . +``` + +The evaluator does not need to inject those variables. An explicit +`FALLTHROUGH_ROUTE_TABLE` remains a strict operator override. The packaged +policy is discarded wholesale only when one of its exact model IDs is absent +from the runtime `ALLOWED_MODELS`; malformed or stale evidence fails closed. + +The JSON Schema documents shape constraints. Runtime validation additionally +checks the canonical route-table hash, runtime source hash, prompt and +normalization policy versions, exact model set, and cross-field task counts. diff --git a/policy/evidence.schema.json b/policy/evidence.schema.json new file mode 100644 index 0000000..a296d84 --- /dev/null +++ b/policy/evidence.schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fallthrough.invalid/schema/packaged-policy-evidence-v1.json", + "title": "Fallthrough packaged measured-policy evidence", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "route_table_sha256", + "runtime_implementation_sha256", + "prompt_policy", + "normalization_policy", + "model_ids", + "model_identity_verified", + "dataset_fingerprints", + "run_fingerprints", + "accuracy_evidence", + "token_evidence" + ], + "properties": { + "schema_version": {"const": 1}, + "route_table_sha256": {"$ref": "#/$defs/sha256"}, + "runtime_implementation_sha256": {"$ref": "#/$defs/sha256"}, + "prompt_policy": {"type": "string", "minLength": 1}, + "normalization_policy": {"type": "string", "minLength": 1}, + "model_ids": { + "type": "array", + "uniqueItems": true, + "items": {"type": "string", "minLength": 1} + }, + "model_identity_verified": {"const": true}, + "dataset_fingerprints": {"$ref": "#/$defs/nonemptySha256Array"}, + "run_fingerprints": {"$ref": "#/$defs/nonemptySha256Array"}, + "accuracy_evidence": { + "type": "object", + "additionalProperties": false, + "required": ["semantics", "task_count", "correct_count"], + "properties": { + "semantics": { + "enum": [ + "official_evaluator", + "transparent_local_proxy_not_official_llm_judge" + ] + }, + "task_count": {"type": "integer", "minimum": 1}, + "correct_count": {"type": "integer", "minimum": 0} + } + }, + "token_evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "complete", + "failed_attempts_included", + "task_count", + "total_fireworks_tokens" + ], + "properties": { + "complete": {"const": true}, + "failed_attempts_included": {"const": true}, + "task_count": {"type": "integer", "minimum": 1}, + "total_fireworks_tokens": {"type": "integer", "minimum": 0} + } + } + }, + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "nonemptySha256Array": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/sha256"} + } + } +} diff --git a/pyproject.toml b/pyproject.toml index a00df81..e758b1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ dependencies = [ ] [project.optional-dependencies] +local = [ + "llama-cpp-python==0.3.33", +] dev = [ "mypy==1.16.1", "pytest==8.4.1", diff --git a/scripts/build_container.sh b/scripts/build_container.sh index 060cfb5..61185bc 100644 --- a/scripts/build_container.sh +++ b/scripts/build_container.sh @@ -1,5 +1,60 @@ #!/usr/bin/env sh set -eu -docker build --no-cache --tag fallthrough . +image="${1:-fallthrough}" +# Treat the organizer's 10 GB cap as decimal GB; this is stricter than 10 GiB. +maximum_compressed_bytes=10000000000 +command -v gzip >/dev/null +command -v python3 >/dev/null +command -v timeout >/dev/null +docker build --no-cache --platform linux/amd64 --tag "$image" . + +architecture="$(docker image inspect --format '{{.Architecture}}' "$image")" +test "$architecture" = "amd64" + +# A cold container plus a trivial offline task is a conservative startup proxy +# for the current deterministic/Fireworks image. If a local route is promoted, +# its GGUF identity+load startup measurement must be checked separately too. +startup_output="$(mktemp)" +trap 'rm -f "$startup_output"' 0 1 2 3 15 +startup_started="$(date +%s)" +printf '[{"task_id":"startup","prompt":"Calculate 1 + 1."}]' | \ + timeout --signal=TERM --kill-after=2s 59s \ + docker run --rm -i \ + --platform linux/amd64 \ + --memory=4000000000 --memory-swap=4000000000 --cpus=2 \ + --network=none \ + --env FALLTHROUGH_POLICY=deterministic-only \ + "$image" --input - --output - \ + >"$startup_output" 2>/dev/null +startup_seconds=$(( $(date +%s) - startup_started )) +test "$startup_seconds" -lt 60 +python3 - "$startup_output" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as handle: + payload = json.load(handle) +expected = [{"task_id": "startup", "answer": "2"}] +if payload != expected: + raise SystemExit(f"unexpected Track 1 output: {payload!r}") +PY +rm -f "$startup_output" +trap - 0 1 2 3 15 + +# A gzipped docker-save stream is a conservative, reproducible local proxy for +# the registry's compressed image size. The published artifact must be checked +# again after push because registries may use different layer compression. +archive="$(mktemp)" +compressed_archive="$archive.gz" +trap 'rm -f "$archive" "$compressed_archive"' 0 1 2 3 15 +docker save --output "$archive" "$image" +gzip -c "$archive" >"$compressed_archive" +compressed_bytes="$(wc -c <"$compressed_archive" | tr -d '[:space:]')" +test "$compressed_bytes" -lt "$maximum_compressed_bytes" +printf 'image=%s architecture=%s compressed_bytes=%s\n' \ + "$image" "$architecture" "$compressed_bytes" +printf 'cold_start_smoke_seconds=%s\n' "$startup_seconds" +rm -f "$archive" "$compressed_archive" +trap - 0 1 2 3 15 diff --git a/scripts/probe_environment.py b/scripts/probe_environment.py index 175c4fe..92c2a69 100644 --- a/scripts/probe_environment.py +++ b/scripts/probe_environment.py @@ -78,7 +78,10 @@ def collect_environment() -> dict[str, Any]: "machine": platform.machine(), }, "pytorch": _torch_report(), - "vllm": {"version": _distribution_version("vllm")}, + "local_runtime": { + "llama_cpp_python_version": _distribution_version("llama-cpp-python"), + }, + "training_runtime": {"vllm_version": _distribution_version("vllm")}, "tools": { "git_available": shutil.which("git") is not None, "docker_available": shutil.which("docker") is not None, diff --git a/scripts/run_container_constrained.sh b/scripts/run_container_constrained.sh new file mode 100644 index 0000000..3d1f29d --- /dev/null +++ b/scripts/run_container_constrained.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env sh +set -eu + +image="${1:-fallthrough}" +input_dir="${FALLTHROUGH_INPUT_DIR:-$PWD/input}" +output_dir="${FALLTHROUGH_OUTPUT_DIR:-$PWD/output}" +result_path="$output_dir/results.json" + +test -f "$input_dir/tasks.json" +mkdir -p "$output_dir" +rm -f "$result_path" + +started_at="$(date +%s)" + +set -- docker run --rm \ + --platform linux/amd64 \ + --memory=4000000000 \ + --memory-swap=4000000000 \ + --cpus=2 \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,size=64m \ + --env FIREWORKS_API_KEY \ + --env FIREWORKS_BASE_URL \ + --env ALLOWED_MODELS \ + --env FALLTHROUGH_GEMMA_MODEL \ + --env FALLTHROUGH_POLICY \ + --env FALLTHROUGH_GLOBAL_TIMEOUT_SECONDS \ + --env FALLTHROUGH_TASK_TIMEOUT_SECONDS \ + --env FALLTHROUGH_OUTPUT_RESERVE_SECONDS \ + --env FALLTHROUGH_CONNECT_TIMEOUT_SECONDS \ + --env FALLTHROUGH_REQUEST_TIMEOUT_SECONDS \ + --env FALLTHROUGH_TRANSPORT_RETRIES \ + --env FALLTHROUGH_FORMAT_RETRIES \ + --env FALLTHROUGH_MAX_CONCURRENCY \ + --env FALLTHROUGH_LOCAL_BACKEND \ + --env FALLTHROUGH_LOCAL_MODEL \ + --env FALLTHROUGH_LOCAL_THREADS \ + --env FALLTHROUGH_LOCAL_CONTEXT_SIZE \ + -v "$input_dir:/input:ro" \ + -v "$output_dir:/output" + +if test -n "${FALLTHROUGH_ROUTE_TABLE_HOST:-}"; then + test -f "$FALLTHROUGH_ROUTE_TABLE_HOST" + route_table_dir="$(CDPATH= cd -P "$(dirname "$FALLTHROUGH_ROUTE_TABLE_HOST")" && pwd)" + route_table_path="$route_table_dir/$(basename "$FALLTHROUGH_ROUTE_TABLE_HOST")" + set -- "$@" \ + -v "$route_table_path:/config/route-table.json:ro" \ + --env FALLTHROUGH_ROUTE_TABLE=/config/route-table.json +fi + +command -v timeout >/dev/null +timeout --signal=TERM --kill-after=5s 599s "$@" "$image" + +elapsed_seconds=$(( $(date +%s) - started_at )) +test "$elapsed_seconds" -lt 600 +test -s "$result_path" +python3 - "$input_dir/tasks.json" "$result_path" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8-sig") as handle: + tasks = json.load(handle) +with open(sys.argv[2], encoding="utf-8") as handle: + results = json.load(handle) + +if not isinstance(tasks, list) or not all(isinstance(task, dict) for task in tasks): + raise SystemExit("Track 1 input must be a JSON array of task objects") +if not isinstance(results, list) or not all(isinstance(row, dict) for row in results): + raise SystemExit("Track 1 output must be a JSON array of result objects") +task_ids = [task.get("task_id") for task in tasks] +result_ids = [row.get("task_id") for row in results] +if any(not isinstance(task_id, str) or not task_id for task_id in task_ids): + raise SystemExit("every Track 1 input requires a nonempty string task_id") +if any(not isinstance(task.get("prompt"), str) or not task["prompt"].strip() for task in tasks): + raise SystemExit("every Track 1 input requires a nonempty string prompt") +if len(set(task_ids)) != len(task_ids): + raise SystemExit("Track 1 input task_id values must be unique") +if result_ids != task_ids: + raise SystemExit("Track 1 output must contain every task_id exactly once and in input order") +for row in results: + if set(row) != {"task_id", "answer"} or not isinstance(row["answer"], str): + raise SystemExit("each Track 1 result must contain only string task_id and answer fields") +PY +printf 'batch_elapsed_seconds=%s result=%s\n' "$elapsed_seconds" "$result_path" diff --git a/scripts/smoke_test_vllm.py b/scripts/smoke_test_llama_cpp.py similarity index 58% rename from scripts/smoke_test_vllm.py rename to scripts/smoke_test_llama_cpp.py index d4a925d..e57c493 100644 --- a/scripts/smoke_test_vllm.py +++ b/scripts/smoke_test_llama_cpp.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Load one optional vLLM model and run a bounded local generation smoke test.""" +"""Load one baked GGUF through llama.cpp and run a bounded local smoke test.""" from __future__ import annotations @@ -10,12 +10,15 @@ import sys import time from collections.abc import Sequence +from pathlib import Path from typing import Any from fallthrough.local import ( - VllmBackend, + LlamaCppBackend, + LocalBackendError, experiment_fingerprint, identify_checkpoint, + source_tree_fingerprint, stable_model_label, unverified_checkpoint_identity, ) @@ -23,24 +26,16 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("model", help="existing model path or locally cached model identifier") + parser.add_argument("model", help="existing, baked .gguf model file") parser.add_argument("--model-label") - parser.add_argument( - "--model-revision", - type=_revision, - help="full 40-digit hex commit required when model is a cached identifier", - ) parser.add_argument("--expected-checkpoint-identity-sha256", type=_sha256) parser.add_argument("--prompt", default="Respond with exactly: OK") parser.add_argument("--max-tokens", type=_positive_int, default=16) - parser.add_argument("--max-model-len", type=_positive_int, default=2048) - parser.add_argument("--dtype", choices=("auto", "half", "float16", "bfloat16"), default="auto") - parser.add_argument("--gpu-memory-utilization", type=_utilization, default=0.82) - parser.add_argument("--enforce-eager", action="store_true") - parser.add_argument( - "--download-dir", - help="local vLLM/Hugging Face cache directory; network access remains disabled", - ) + parser.add_argument("--context-size", type=_context_size, default=1024) + parser.add_argument("--threads", type=_threads, default=2) + parser.add_argument("--batch-size", type=_batch_size, default=128) + parser.add_argument("--load-timeout-seconds", type=_load_timeout, default=50.0) + parser.add_argument("--generation-timeout-seconds", type=_generation_timeout, default=25.0) return parser.parse_args(argv) @@ -51,13 +46,40 @@ def _positive_int(value: str) -> int: return parsed -def _utilization(value: str) -> float: +def _bounded_int(value: str, minimum: int, maximum: int, label: str) -> int: + parsed = int(value) + if not minimum <= parsed <= maximum: + raise argparse.ArgumentTypeError(f"{label} must be between {minimum} and {maximum}") + return parsed + + +def _context_size(value: str) -> int: + return _bounded_int(value, 128, 4096, "context size") + + +def _threads(value: str) -> int: + return _bounded_int(value, 1, 2, "threads") + + +def _batch_size(value: str) -> int: + return _bounded_int(value, 1, 512, "batch size") + + +def _bounded_float(value: str, maximum: float, label: str) -> float: parsed = float(value) - if not math.isfinite(parsed) or not 0.0 < parsed <= 1.0: - raise argparse.ArgumentTypeError("must be finite and in (0, 1]") + if not math.isfinite(parsed) or not 0.0 < parsed <= maximum: + raise argparse.ArgumentTypeError(f"{label} must be finite, positive, and <= {maximum}") return parsed +def _load_timeout(value: str) -> float: + return _bounded_float(value, 55.0, "load timeout") + + +def _generation_timeout(value: str) -> float: + return _bounded_float(value, 25.0, "generation timeout") + + def _sha256(value: str) -> str: normalized = value.casefold() if len(normalized) != 64 or any( @@ -67,15 +89,6 @@ def _sha256(value: str) -> str: return normalized -def _revision(value: str) -> str: - normalized = value.casefold() - if len(normalized) != 40 or any( - character not in "0123456789abcdef" for character in normalized - ): - raise argparse.ArgumentTypeError("must be a full 40-digit hexadecimal commit") - return normalized - - def _generation_configuration(args: argparse.Namespace) -> dict[str, object]: return { "max_tokens": args.max_tokens, @@ -89,7 +102,7 @@ def run_smoke(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: identity_started = time.perf_counter() identity_error: BaseException | None = None try: - checkpoint = identify_checkpoint(args.model, args.model_revision) + checkpoint = identify_checkpoint(args.model) except (OSError, ValueError) as exc: checkpoint = unverified_checkpoint_identity(args.model) identity_error = exc @@ -100,31 +113,39 @@ def run_smoke(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: checkpoint.identity_sha256, ) planned_configuration: dict[str, object] = { - "dtype": args.dtype, - "enforce_eager": args.enforce_eager, - "generation_config": "vllm", - "gpu_memory_utilization": args.gpu_memory_utilization, - "load_format": "safetensors", - "max_model_len": args.max_model_len, - "revision_bound": args.model_revision is not None, - "tensor_parallel_size": 1, - "vllm_version": None, + "batch_size": args.batch_size, + "chat_template_source": "gguf_metadata", + "context_size": args.context_size, + "llama_cpp_python_version": None, + "n_gpu_layers": 0, + "seed": 0, + "threads": args.threads, + "use_mlock": False, + "use_mmap": True, } generation_configuration = _generation_configuration(args) - planned_hardware: dict[str, object] = {} + implementation_id = source_tree_fingerprint(Path(__file__)) + planned_configuration["load_timeout_seconds"] = args.load_timeout_seconds + planned_configuration["generation_timeout_seconds"] = args.generation_timeout_seconds + planned_hardware: dict[str, object] = {"threads_used": args.threads} report: dict[str, Any] = { - "backend": "vllm", + "backend": "llama_cpp", "model": model_label, "run_fingerprint": experiment_fingerprint( - backend="vllm", + backend="llama_cpp", model_label=model_label, checkpoint_identity_sha256=checkpoint.identity_sha256, + checkpoint_kind=checkpoint.kind, + checkpoint_file_count=checkpoint.file_count, + checkpoint_total_bytes=checkpoint.total_bytes, configuration={ "backend": planned_configuration, "generation": generation_configuration, }, hardware=planned_hardware, + implementation_sha256=implementation_id, ), + "implementation_sha256": implementation_id, "checkpoint_kind": checkpoint.kind, "checkpoint_identity_sha256": checkpoint.identity_sha256, "checkpoint_file_count": checkpoint.file_count, @@ -136,42 +157,46 @@ def run_smoke(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: "model_load_success": False, "generation_success": False, "load_time_seconds": None, + "startup_time_seconds": None, "generation_latency_ms": None, "prompt_tokens": None, "tokens_generated": None, "finish_reason": None, - "load_peak_vram_bytes": None, - "generation_peak_vram_bytes": None, - "observed_peak_vram_bytes": None, - "vram_measurement": "observed_global_device_0_peak_bytes", + "load_peak_memory_bytes": None, + "generation_peak_memory_bytes": None, + "observed_peak_memory_bytes": None, + "memory_measurement": "container_cgroup_peak_or_worker_process_peak_rss_bytes", "error_code": None, "error_type": None, "error": None, } if identity_error is not None: + report["startup_time_seconds"] = time.perf_counter() - identity_started report["error_code"] = "invalid_checkpoint_identity" report["error_type"] = type(identity_error).__name__ report["error"] = "checkpoint identity validation failed" return 1, report expected_sha256 = args.expected_checkpoint_identity_sha256 if expected_sha256 is not None and expected_sha256 != checkpoint.identity_sha256: + report["startup_time_seconds"] = time.perf_counter() - identity_started report["error_code"] = "checkpoint_identity_mismatch" report["error"] = "checkpoint identity SHA-256 mismatch" return 1, report - backend: VllmBackend | None = None + + backend: LlamaCppBackend | None = None load_started = time.perf_counter() try: - backend = VllmBackend.load( + backend = LlamaCppBackend.load( args.model, - dtype=args.dtype, - gpu_memory_utilization=args.gpu_memory_utilization, - max_model_len=args.max_model_len, - enforce_eager=args.enforce_eager, - download_dir=args.download_dir, - revision=args.model_revision, + threads=args.threads, + context_size=args.context_size, + batch_size=args.batch_size, + load_timeout_seconds=args.load_timeout_seconds, + generation_timeout_seconds=args.generation_timeout_seconds, ) except Exception as exc: report["load_time_seconds"] = time.perf_counter() - load_started + report["startup_time_seconds"] = time.perf_counter() - identity_started report["error_code"] = "local_model_load_failed" report["error_type"] = type(exc).__name__ report["error"] = "local model load failed" @@ -185,14 +210,19 @@ def run_smoke(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: backend=backend.backend_name, model_label=model_label, checkpoint_identity_sha256=checkpoint.identity_sha256, + checkpoint_kind=checkpoint.kind, + checkpoint_file_count=checkpoint.file_count, + checkpoint_total_bytes=checkpoint.total_bytes, configuration={ "backend": backend.configuration, "generation": generation_configuration, }, hardware=backend.hardware, + implementation_sha256=implementation_id, ) report["load_time_seconds"] = backend.load_time_seconds - report["load_peak_vram_bytes"] = backend.load_peak_vram_bytes + report["startup_time_seconds"] = time.perf_counter() - identity_started + report["load_peak_memory_bytes"] = backend.load_peak_memory_bytes generation = backend.generate_sync( args.prompt, max_tokens=args.max_tokens, @@ -202,14 +232,14 @@ def run_smoke(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: report["prompt_tokens"] = generation.prompt_token_count report["tokens_generated"] = generation.token_count report["finish_reason"] = generation.finish_reason - report["generation_peak_vram_bytes"] = generation.peak_vram_bytes - report["vram_measurement"] = backend.vram_measurement + report["generation_peak_memory_bytes"] = generation.peak_memory_bytes + report["memory_measurement"] = backend.memory_measurement peaks = [ value - for value in (backend.load_peak_vram_bytes, generation.peak_vram_bytes) + for value in (backend.load_peak_memory_bytes, generation.peak_memory_bytes) if isinstance(value, int) ] - report["observed_peak_vram_bytes"] = max(peaks) if peaks else None + report["observed_peak_memory_bytes"] = max(peaks) if peaks else None if not generation.text.strip(): report["error_code"] = "empty_completion" report["error"] = "local generation returned an empty completion" @@ -219,14 +249,18 @@ def run_smoke(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: report["error"] = f"local generation ended with {generation.finish_reason}" return 1, report report["generation_success"] = True - if generation.latency_ms > 0: - report["generated_tokens_per_second"] = round( - generation.token_count / (generation.latency_ms / 1000.0), 3 - ) - else: - report["generated_tokens_per_second"] = None + report["generated_tokens_per_second"] = ( + round(generation.token_count / (generation.latency_ms / 1000.0), 3) + if generation.latency_ms > 0 + else None + ) return 0, report except Exception as exc: + if isinstance(exc, LocalBackendError): + report["generation_latency_ms"] = exc.latency_ms + report["generation_peak_memory_bytes"] = exc.peak_memory_bytes + if exc.memory_measurement is not None: + report["memory_measurement"] = exc.memory_measurement report["error_code"] = "local_generation_failed" report["error_type"] = type(exc).__name__ report["error"] = "local generation failed" diff --git a/src/fallthrough/config.py b/src/fallthrough/config.py index d954da2..5b53964 100644 --- a/src/fallthrough/config.py +++ b/src/fallthrough/config.py @@ -13,6 +13,7 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from urllib.parse import urlsplit class ConfigurationError(ValueError): @@ -44,6 +45,12 @@ def _parse_models(raw: str | None) -> tuple[str, ...]: return tuple(dict.fromkeys(value.strip() for value in values if value.strip())) +def _is_gemma_model(model: str) -> bool: + """Identify a Gemma model without depending on a launch-day model list.""" + + return "gemma" in model.casefold() + + def _float(environment: Mapping[str, str], name: str, default: float) -> float: raw = environment.get(name) if raw is None: @@ -90,7 +97,11 @@ class Settings: input_path: Path | None = None output_path: Path | None = None trace_path: Path | None = None + # Explicit developer/evaluator override. It is always strict. route_table_path: Path | None = None + # Image-owned measured policy. It may fall back only on allowlist incompatibility. + packaged_route_table_path: Path | None = None + packaged_policy_evidence_path: Path | None = None fireworks_base_url: str | None = None fireworks_api_key: str | None = None @@ -99,12 +110,15 @@ class Settings: gemma_model: str | None = None local_backend: str = "none" local_model: str | None = None + local_threads: int = 2 + local_context_size: int = 1024 policy: str = "hybrid-baseline" global_timeout_seconds: float = 270.0 + task_timeout_seconds: float = 29.0 output_reserve_seconds: float = 2.0 connect_timeout_seconds: float = 5.0 - request_timeout_seconds: float = 30.0 + request_timeout_seconds: float = 25.0 transport_retries: int = 1 # Validation-gated corrective retries per remote call (0 disables). format_retries: int = 1 @@ -126,6 +140,13 @@ def from_env(cls, environment: Mapping[str, str] | None = None) -> Settings: output_path = _first(env, "FALLTHROUGH_OUTPUT_PATH") trace_path = _first(env, "FALLTHROUGH_TRACE_PATH") route_table_path = _first(env, "FALLTHROUGH_ROUTE_TABLE") + packaged_route_table_path = _first(env, "FALLTHROUGH_PACKAGED_ROUTE_TABLE") + packaged_policy_evidence_path = _first(env, "FALLTHROUGH_PACKAGED_POLICY_EVIDENCE") + if bool(packaged_route_table_path) != bool(packaged_policy_evidence_path): + raise ConfigurationError( + "FALLTHROUGH_PACKAGED_ROUTE_TABLE and " + "FALLTHROUGH_PACKAGED_POLICY_EVIDENCE must be configured together" + ) conventional_input = Path("/input/tasks.json") uses_conventional_paths = input_path is None and conventional_input.is_file() if uses_conventional_paths: @@ -143,30 +164,44 @@ def from_env(cls, environment: Mapping[str, str] | None = None) -> Settings: "FALLTHROUGH_POLICY must be hybrid-baseline, gemma-only, or deterministic-only" ) local_backend = env.get("FALLTHROUGH_LOCAL_BACKEND", "none").strip().lower() or "none" - if local_backend not in {"none", "vllm"}: - raise ConfigurationError("FALLTHROUGH_LOCAL_BACKEND must be none or vllm") - - raw_allowed = ( - env.get("ALLOWED_MODELS") - if "ALLOWED_MODELS" in env - else env.get("FIREWORKS_ALLOWED_MODELS") - ) - allowed_declared = any( - name in env for name in ("ALLOWED_MODELS", "FIREWORKS_ALLOWED_MODELS") + if local_backend not in {"none", "llama_cpp"}: + raise ConfigurationError("FALLTHROUGH_LOCAL_BACKEND must be none or llama_cpp") + local_threads = _integer(env, "FALLTHROUGH_LOCAL_THREADS", 2, minimum=1) + if local_threads > 2: + raise ConfigurationError("FALLTHROUGH_LOCAL_THREADS cannot exceed the grader's 2 vCPU") + local_context_size = _integer( + env, + "FALLTHROUGH_LOCAL_CONTEXT_SIZE", + 1024, + minimum=128, ) + if local_context_size > 4096: + raise ConfigurationError("FALLTHROUGH_LOCAL_CONTEXT_SIZE cannot exceed 4096") + + # The final evaluator contract names ALLOWED_MODELS explicitly. Do not + # let a legacy alias turn a missing official allowlist into an + # allow-all configuration. + raw_allowed = env.get("ALLOWED_MODELS") + allowed_declared = "ALLOWED_MODELS" in env allowed = _parse_models(raw_allowed) - explicit_gemma = ( - env.get("FALLTHROUGH_GEMMA_MODEL", "").strip() or None - if "FALLTHROUGH_GEMMA_MODEL" in env - else _first(env, "FIREWORKS_GEMMA_MODEL") - ) - gemma = explicit_gemma or next( - (model for model in allowed if "gemma" in model.casefold()), None - ) + explicit_gemma = env.get("FALLTHROUGH_GEMMA_MODEL", "").strip() or None + if explicit_gemma is not None and not _is_gemma_model(explicit_gemma): + raise ConfigurationError("FALLTHROUGH_GEMMA_MODEL must identify a Gemma model") + gemma = explicit_gemma or next((model for model in allowed if _is_gemma_model(model)), None) if gemma and allowed_declared and gemma not in allowed: raise ConfigurationError("configured Gemma model is not present in ALLOWED_MODELS") global_timeout = _float(env, "FALLTHROUGH_GLOBAL_TIMEOUT_SECONDS", 270.0) + if global_timeout >= 600.0: + raise ConfigurationError("FALLTHROUGH_GLOBAL_TIMEOUT_SECONDS must be below 600") + task_timeout = _float(env, "FALLTHROUGH_TASK_TIMEOUT_SECONDS", 29.0) + if task_timeout >= 30.0: + raise ConfigurationError("FALLTHROUGH_TASK_TIMEOUT_SECONDS must be below 30") + request_timeout = _float(env, "FALLTHROUGH_REQUEST_TIMEOUT_SECONDS", 25.0) + if request_timeout > task_timeout: + raise ConfigurationError( + "FALLTHROUGH_REQUEST_TIMEOUT_SECONDS cannot exceed the task timeout" + ) output_reserve = _nonnegative_float(env, "FALLTHROUGH_OUTPUT_RESERVE_SECONDS", 2.0) if output_reserve >= global_timeout: raise ConfigurationError( @@ -178,6 +213,12 @@ def from_env(cls, environment: Mapping[str, str] | None = None) -> Settings: output_path=Path(output_path) if output_path and output_path != "-" else None, trace_path=Path(trace_path) if trace_path and trace_path != "-" else None, route_table_path=Path(route_table_path) if route_table_path else None, + packaged_route_table_path=( + Path(packaged_route_table_path) if packaged_route_table_path else None + ), + packaged_policy_evidence_path=( + Path(packaged_policy_evidence_path) if packaged_policy_evidence_path else None + ), fireworks_base_url=_first(env, "FIREWORKS_BASE_URL"), fireworks_api_key=_first(env, "FIREWORKS_API_KEY"), allowed_models=allowed, @@ -185,11 +226,14 @@ def from_env(cls, environment: Mapping[str, str] | None = None) -> Settings: gemma_model=gemma, local_backend=local_backend, local_model=_first(env, "FALLTHROUGH_LOCAL_MODEL"), + local_threads=local_threads, + local_context_size=local_context_size, policy=policy, global_timeout_seconds=global_timeout, + task_timeout_seconds=task_timeout, output_reserve_seconds=output_reserve, connect_timeout_seconds=_float(env, "FALLTHROUGH_CONNECT_TIMEOUT_SECONDS", 5.0), - request_timeout_seconds=_float(env, "FALLTHROUGH_REQUEST_TIMEOUT_SECONDS", 30.0), + request_timeout_seconds=request_timeout, transport_retries=_integer(env, "FALLTHROUGH_TRANSPORT_RETRIES", 1), format_retries=_integer(env, "FALLTHROUGH_FORMAT_RETRIES", 1), max_concurrency=_integer(env, "FALLTHROUGH_MAX_CONCURRENCY", 8, minimum=1), @@ -216,15 +260,31 @@ def validate_fireworks(self, *, require_gemma: bool = False) -> None: missing.append("FIREWORKS_BASE_URL") if not self.fireworks_api_key: missing.append("FIREWORKS_API_KEY") + if not self.allowed_models_configured: + missing.append("ALLOWED_MODELS") if require_gemma and not self.gemma_model: missing.append("FALLTHROUGH_GEMMA_MODEL or a Gemma entry in ALLOWED_MODELS") if missing: route = "remote Gemma route" if require_gemma else "Fireworks route" raise ConfigurationError(route + " requires " + ", ".join(missing)) + if not self.allowed_models: + raise ConfigurationError("ALLOWED_MODELS must contain at least one model") + if self.gemma_model is not None and not _is_gemma_model(self.gemma_model): + raise ConfigurationError("configured Gemma model must identify a Gemma model") if ( - require_gemma - and self.gemma_model is not None + self.gemma_model is not None and self.allowed_models_configured and self.gemma_model not in self.allowed_models ): raise ConfigurationError("configured Gemma model is not present in ALLOWED_MODELS") + assert self.fireworks_base_url is not None + try: + parsed_url = urlsplit(self.fireworks_base_url) + hostname = parsed_url.hostname + port = parsed_url.port + except ValueError as exc: + raise ConfigurationError("FIREWORKS_BASE_URL is not a valid URL") from exc + if parsed_url.scheme not in {"http", "https"} or not hostname: + raise ConfigurationError("FIREWORKS_BASE_URL must be an absolute HTTP(S) URL") + if port is not None and not 1 <= port <= 65535: + raise ConfigurationError("FIREWORKS_BASE_URL has an invalid port") diff --git a/src/fallthrough/fireworks/client.py b/src/fallthrough/fireworks/client.py index 61f7e02..97a48b1 100644 --- a/src/fallthrough/fireworks/client.py +++ b/src/fallthrough/fireworks/client.py @@ -31,7 +31,7 @@ def __init__( self, base_url: str, api_key: str, - allowed_models: Collection[str] | str | None = None, + allowed_models: Collection[str] | str, *, connect_timeout_s: float = 5.0, read_timeout_s: float = 30.0, @@ -48,7 +48,7 @@ def __init__( code="invalid_api_key", ) self._api_key = api_key.strip() - self._allowed_models = None if allowed_models is None else _normalise_models(allowed_models) + self._allowed_models = _normalise_models(allowed_models) if isinstance(max_retries, bool) or not isinstance(max_retries, int) or max_retries < 0: raise FireworksConfigurationError( @@ -100,11 +100,11 @@ def endpoint_url(self) -> str: @property def allowed_models(self) -> frozenset[str]: - return self._allowed_models or frozenset() + return self._allowed_models @property def allowlist_enforced(self) -> bool: - return self._allowed_models is not None + return True async def complete( self, @@ -113,6 +113,7 @@ async def complete( *, max_tokens: int, temperature: float = 0.0, + reasoning_effort: str | int | bool | None = None, ) -> FireworksResponse: """Generate a non-streaming completion for one compact user prompt.""" @@ -121,6 +122,7 @@ async def complete( messages=({"role": "user", "content": prompt},), max_tokens=max_tokens, temperature=temperature, + reasoning_effort=reasoning_effort, ) async def chat_completion( @@ -130,11 +132,12 @@ async def chat_completion( messages: Sequence[Mapping[str, Any]], max_tokens: int, temperature: float = 0.0, + reasoning_effort: str | int | bool | None = None, ) -> FireworksResponse: """Call the OpenAI-compatible chat completions endpoint.""" self.validate_model(model) - _validate_generation_options(messages, max_tokens, temperature) + _validate_generation_options(messages, max_tokens, temperature, reasoning_effort) payload = { "model": model, "messages": [dict(message) for message in messages], @@ -142,6 +145,8 @@ async def chat_completion( "temperature": temperature, "stream": False, } + if reasoning_effort is not None: + payload["reasoning_effort"] = reasoning_effort headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", @@ -255,7 +260,7 @@ def validate_model(self, model: str) -> None: "Fireworks model must not be empty", code="invalid_model", ) - if self._allowed_models is not None and model not in self._allowed_models: + if model not in self._allowed_models: raise FireworksModelNotAllowedError( f"Fireworks model is not allowed: {model}", code="model_not_allowed", @@ -292,11 +297,17 @@ def _chat_completions_url(base_url: str) -> httpx.URL: ) try: url = httpx.URL(base_url.strip()) - except (TypeError, ValueError) as exc: + port = url.port + except (TypeError, ValueError, httpx.InvalidURL) as exc: raise FireworksConfigurationError( "Fireworks base URL is invalid", code="invalid_base_url", ) from exc + if port is not None and not 1 <= port <= 65535: + raise FireworksConfigurationError( + "Fireworks base URL has an invalid port", + code="invalid_base_url", + ) if url.scheme not in ("http", "https") or not url.host: raise FireworksConfigurationError( "Fireworks base URL must be an absolute HTTP(S) URL", @@ -310,10 +321,8 @@ def _chat_completions_url(base_url: str) -> httpx.URL: def _normalise_models( - allowed_models: Collection[str] | str | None, + allowed_models: Collection[str] | str, ) -> frozenset[str]: - if allowed_models is None: - return frozenset() values: Collection[str] if isinstance(allowed_models, str): values = allowed_models.split(",") @@ -326,6 +335,7 @@ def _validate_generation_options( messages: Sequence[Mapping[str, Any]], max_tokens: int, temperature: float, + reasoning_effort: str | int | bool | None, ) -> None: if not messages: raise FireworksConfigurationError( @@ -362,6 +372,19 @@ def _validate_generation_options( "temperature must be a non-negative number", code="invalid_temperature", ) + valid_efforts = {"none", "low", "medium", "high", "xhigh", "max"} + if isinstance(reasoning_effort, str): + if reasoning_effort not in valid_efforts: + raise FireworksConfigurationError( + "reasoning_effort is not supported", + code="invalid_reasoning_effort", + ) + elif reasoning_effort is not None and not isinstance(reasoning_effort, bool): + if not isinstance(reasoning_effort, int) or reasoning_effort <= 0: + raise FireworksConfigurationError( + "reasoning_effort must be a supported string, boolean, or positive integer", + code="invalid_reasoning_effort", + ) def _parse_completion( diff --git a/src/fallthrough/inference.py b/src/fallthrough/inference.py index e9b26b9..f5dc468 100644 --- a/src/fallthrough/inference.py +++ b/src/fallthrough/inference.py @@ -6,8 +6,16 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Protocol -from fallthrough.tasks.labels import sentiment_labels_from_metadata -from fallthrough.validation.sentiment import requested_sentiment_labels +from fallthrough.solvers.arithmetic import solve_arithmetic +from fallthrough.tasks.labels import sentiment_instruction_header, sentiment_labels_from_metadata +from fallthrough.tasks.requirements import ( + requests_fenced_output, + requests_non_english_output, +) +from fallthrough.validation.sentiment import ( + requested_sentiment_labels, + sentiment_reason_required, +) if TYPE_CHECKING: from fallthrough.tasks.kinds import TaskKind @@ -22,68 +30,79 @@ def metadata(self) -> Mapping[str, Any]: ... DEFAULT_MAX_TOKENS_BY_KIND: dict[str, int] = { - "arithmetic": 32, - "sentiment": 8, - "entity_extraction": 128, + # These are safety ceilings, not requested output lengths. The retired + # organizer examples include multi-step word problems and explanatory + # factual/sentiment answers; 64-128 token ceilings truncated otherwise + # correct Kimi development responses. Fireworks charges actual usage, so a + # larger ceiling prevents accuracy failures without charging unused tokens. + "arithmetic": 256, + "sentiment": 128, + "entity_extraction": 256, "summarisation": 256, - "factual_qa": 32, - "logic": 64, + "factual_qa": 256, + "logic": 256, "code_debug": 512, "code_generation": 512, "structured": 256, - "unknown": 256, + "unknown": 512, } -_INSTRUCTIONS: dict[str, str] = { - "arithmetic": "Return only the final answer. No explanation.", - "sentiment": "Return exactly one requested sentiment label.", - "entity_extraction": "Return only the requested entities in the required format.", - "summarisation": "Return only the requested summary.", - "factual_qa": "Answer only. Maximum 8 words.", - "logic": "Return only the requested final answer. No explanation.", - "code_debug": "Return only the requested corrected code.", - "code_generation": "Return only the requested code.", - "structured": "Return only the requested output in the required format.", - "unknown": "Return only the requested answer.", -} +PROMPT_POLICY = "contract-preserving-selective-v3" +NORMALIZATION_POLICY = "contract-preserving-v2" +KIMI_CONCISION_SUFFIX = "\n\nBe concise. Preserve all major points and every stated constraint." +KIMI_PROMPT_POLICY = "selective-concision-v1" + +_EXPLANATORY_FACTUAL_RE = re.compile( + r"\b(?:compare|contrast|describe|difference|differentiate|explain|how|why)\b", + re.IGNORECASE, +) + + +def kimi_prompt_suffix(task: _TaskLike, kind: TaskKind) -> str: + """Add a compactness reminder only where development evidence supports it. + + Short objective tasks already elicit compact answers, so an unconditional + suffix only adds scored prompt tokens. Retired organizer examples showed a + benefit for explanatory facts, multi-step math that the deterministic + solver cannot prove, and long summaries where preserving all major points + matters. The rule depends on task shape, never prompt identity. + """ + + kind_value = getattr(kind, "value", str(kind)) + prompt = task.prompt + if kind_value == "factual_qa" and _EXPLANATORY_FACTUAL_RE.search(prompt): + return KIMI_CONCISION_SUFFIX + if kind_value == "arithmetic" and solve_arithmetic(task) is None: + return KIMI_CONCISION_SUFFIX + if kind_value == "summarisation" and len(prompt) >= 350: + return KIMI_CONCISION_SUFFIX + return "" + _FULL_CODE_FENCE_RE = re.compile( r"\A\s*```[A-Za-z0-9_+.-]*[ \t]*\r?\n(?P.*?)\r?\n```\s*\Z", re.DOTALL, ) -_NEGATED_FENCE_REQUEST_RE = re.compile( - r"\b(?:do\s+not|don't|without|no|never|avoid|omit)\b[^.\n]{0,60}" - r"(?:\b(?:a\s+)?(?:markdown\s+)?code\s+(?:block|fence)s?\b|" - r"\bfenced\s+code(?:\s+block)?\b|\btriple\s+backticks?\b)", - re.IGNORECASE, -) -_POSITIVE_FENCE_REQUEST_RE = re.compile( - r"\b(?:write|return|output|provide|format|wrap|enclose|put|generate|create)\b" - r"[^.\n]{0,100}\b(?:in|inside|within|as|using|with)\s+(?:a\s+)?" - r"(?:(?:markdown\s+)?code\s+(?:block|fence)|fenced\s+code(?:\s+block)?|" - r"triple\s+backticks?)\b|" - r"(?:^|[.;]\s*)(?:return|output|provide|include|wrap|enclose)\s+" - r"(?:only\s+)?(?:the\s+|a\s+)?" - r"(?:(?:markdown\s+)?code\s+(?:block|fence)s?|fenced\s+code(?:\s+block)?|" - r"triple\s+backticks?)\b", - re.IGNORECASE | re.MULTILINE, -) -_CODE_SOURCE_DELIMITER_RE = re.compile( - r"\b(?:code|source|input|snippet|program)\s*:", - re.IGNORECASE, -) def build_task_prompt(task: _TaskLike, kind: TaskKind) -> str: - """Build a compact instruction while preserving task constraints.""" + """Use the evaluator prompt verbatim unless trusted metadata adds a constraint. + + Prompt tokens are scored, and a synthetic category instruction must never + narrow an evaluator request for an explanation, reason, or exact shape. + """ kind_value = getattr(kind, "value", str(kind)) - instruction = _INSTRUCTIONS.get(kind_value, _INSTRUCTIONS["unknown"]) + prompt = task.prompt if kind_value == "sentiment": labels = sentiment_labels_from_metadata(task.metadata) if labels: - instruction = f"Return exactly one label: {' | '.join(labels)}." - return f"{instruction}\n\n{task.prompt}" + header = sentiment_instruction_header(task.prompt).casefold() + if not all(label.casefold() in header for label in labels): + prompt = f"{prompt}\n\nAllowed labels: {' | '.join(labels)}." + if requests_non_english_output(task.prompt): + prompt = f"{prompt}\n\nAnswer in English." + return prompt prompt_for_task = build_task_prompt @@ -101,7 +120,7 @@ def normalize_code_answer(task: _TaskLike, kind: TaskKind, answer: str) -> str: code = match.group("code") if not code.strip(): return "" - return answer if _requests_fenced_output(task.prompt) else code + return answer if requests_fenced_output(task.prompt) else code _ANSWER_PREFIX_RE = re.compile( @@ -109,18 +128,14 @@ def normalize_code_answer(task: _TaskLike, kind: TaskKind, answer: str) -> str: re.IGNORECASE, ) _FINAL_NUMBER_RE = re.compile(r"[+-]?\d[\d,_]*(?:\.\d+)?(?:\s*/\s*\d+)?%?") -# Kinds whose graded answer is one short line; a trailing explanation line is -# presentation, and the last non-empty line carries the actual answer. -_SINGLE_LINE_KIND_VALUES = frozenset({"factual_qa", "sentiment", "arithmetic", "logic"}) def canonicalize_answer(task: _TaskLike, kind: TaskKind, answer: str) -> str: - """Strip presentation wrappers so exact-match grading sees the answer itself. + """Apply only transformations that cannot discard requested answer content. - Purely local text canonicalization — never invents content. Code kinds - keep the existing fence handling; summarisation, entity extraction, and - structured output pass through (minus a whole-answer presentation fence) - because reformatting them risks destroying the requested shape. + Code keeps its fence handling. Objective one-expression arithmetic and + label-only sentiment can be normalized safely; word-problem work, factual + explanations, logic reasoning, and sentiment reasons remain intact. """ kind_value = getattr(kind, "value", str(kind)) @@ -128,27 +143,27 @@ def canonicalize_answer(task: _TaskLike, kind: TaskKind, answer: str) -> str: if kind_value in {"code_debug", "code_generation"}: return normalize_code_answer(task, kind, text) fence = _FULL_CODE_FENCE_RE.fullmatch(text) - if fence is not None and not _requests_fenced_output(task.prompt): + if fence is not None and not requests_fenced_output(task.prompt): text = fence.group("code").strip() - if kind_value not in _SINGLE_LINE_KIND_VALUES: - return text - - lines = [line.strip() for line in text.splitlines() if line.strip()] - if lines: - text = lines[-1] - text = _ANSWER_PREFIX_RE.sub("", text).strip() - if len(text) >= 2 and text[0] == text[-1] and text[0] in {'"', "'"}: - text = text[1:-1].strip() - - if kind_value == "arithmetic": + if kind_value == "arithmetic" and solve_arithmetic(task) is not None: + lines = [line.strip() for line in text.splitlines() if line.strip()] + if lines: + text = lines[-1] + text = _ANSWER_PREFIX_RE.sub("", text).strip() numbers: list[str] = _FINAL_NUMBER_RE.findall(text) if numbers: return numbers[-1].replace(",", "").replace("_", "").replace(" ", "") return text.rstrip(".") - if kind_value == "sentiment": + if kind_value == "sentiment" and not sentiment_reason_required(task): + lines = [line.strip() for line in text.splitlines() if line.strip()] + if lines: + text = lines[-1] + text = _ANSWER_PREFIX_RE.sub("", text).strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in {'"', "'"}: + text = text[1:-1].strip() return _canonical_sentiment_label(task, text) - if text.endswith(".") and len(text) < 120: - text = text[:-1] + # Stripping prose after generation cannot save scored tokens; it can only + # remove content the LLM accuracy judge requires. return text @@ -183,19 +198,3 @@ def corrective_prompt(task: _TaskLike, kind: TaskKind, reason: str) -> str: f"Your previous answer was rejected ({reason}). " "Answer again and follow the required output format exactly." ) - - -def _requests_fenced_output(prompt: str) -> bool: - source = _CODE_SOURCE_DELIMITER_RE.search(prompt) - newline = re.search(r"\r?\n", prompt) - cutoff = len(prompt) - if source is not None: - cutoff = min(cutoff, source.start()) - if newline is not None: - cutoff = min(cutoff, newline.start()) - instruction = prompt[:cutoff] - negative = _NEGATED_FENCE_REQUEST_RE.search(instruction) - positive = _POSITIVE_FENCE_REQUEST_RE.search(instruction) - if positive is None: - return False - return negative is None or positive.start() < negative.start() diff --git a/src/fallthrough/local/__init__.py b/src/fallthrough/local/__init__.py index d543a5d..6a15ae9 100644 --- a/src/fallthrough/local/__init__.py +++ b/src/fallthrough/local/__init__.py @@ -1,23 +1,35 @@ """Optional in-container local inference backends.""" from .backend import LocalBackend, LocalBackendError, LocalGeneration +from .gguf_metadata import ( + GGUFModelInfo, + inspect_gguf, + nominal_quantization_bits, + tensor_type_evidence_consistent, +) from .identity import ( CheckpointIdentity, experiment_fingerprint, identify_checkpoint, + source_tree_fingerprint, stable_model_label, unverified_checkpoint_identity, ) -from .vllm_backend import VllmBackend +from .llama_cpp_backend import LlamaCppBackend __all__ = [ "LocalBackend", "LocalBackendError", "LocalGeneration", "CheckpointIdentity", - "VllmBackend", + "GGUFModelInfo", + "LlamaCppBackend", "experiment_fingerprint", "identify_checkpoint", + "inspect_gguf", + "nominal_quantization_bits", + "source_tree_fingerprint", "stable_model_label", + "tensor_type_evidence_consistent", "unverified_checkpoint_identity", ] diff --git a/src/fallthrough/local/backend.py b/src/fallthrough/local/backend.py index c50f5d5..36b2d4c 100644 --- a/src/fallthrough/local/backend.py +++ b/src/fallthrough/local/backend.py @@ -10,6 +10,19 @@ class LocalBackendError(RuntimeError): """Raised when optional local model loading or generation fails.""" + def __init__( + self, + message: str, + *, + latency_ms: float | None = None, + peak_memory_bytes: int | None = None, + memory_measurement: str | None = None, + ) -> None: + super().__init__(message) + self.latency_ms = latency_ms + self.peak_memory_bytes = peak_memory_bytes + self.memory_measurement = memory_measurement + @dataclass(frozen=True, slots=True) class LocalGeneration: @@ -19,7 +32,7 @@ class LocalGeneration: token_count: int latency_ms: float prompt_token_count: int | None = None - peak_vram_bytes: int | None = None + peak_memory_bytes: int | None = None finish_reason: str | None = None metadata: dict[str, Any] = field(default_factory=dict) @@ -44,11 +57,13 @@ def __post_init__(self) -> None: or self.latency_ms < 0 ): raise ValueError("latency_ms must be a finite non-negative number") - if self.peak_vram_bytes is not None: - if isinstance(self.peak_vram_bytes, bool) or not isinstance(self.peak_vram_bytes, int): - raise TypeError("peak_vram_bytes must be an integer") - if self.peak_vram_bytes < 0: - raise ValueError("peak_vram_bytes cannot be negative") + if self.peak_memory_bytes is not None: + if isinstance(self.peak_memory_bytes, bool) or not isinstance( + self.peak_memory_bytes, int + ): + raise TypeError("peak_memory_bytes must be an integer") + if self.peak_memory_bytes < 0: + raise ValueError("peak_memory_bytes cannot be negative") class LocalBackend(Protocol): @@ -57,8 +72,8 @@ class LocalBackend(Protocol): backend_name: str model: str load_time_seconds: float - load_peak_vram_bytes: int | None - vram_measurement: str + load_peak_memory_bytes: int | None + memory_measurement: str configuration: dict[str, object] hardware: dict[str, object] diff --git a/src/fallthrough/local/gguf_metadata.py b/src/fallthrough/local/gguf_metadata.py new file mode 100644 index 0000000..46bb30c --- /dev/null +++ b/src/fallthrough/local/gguf_metadata.py @@ -0,0 +1,398 @@ +"""Bounded, little-endian GGUF v2/v3 artifact inspection. + +The reader touches metadata and tensor descriptors only. It never reads tensor +payloads, downloads files, or trusts model filenames. GGUF v3 can technically +be big-endian, but the format has no byte-order marker; such files fail closed +as an unsupported version here. +""" + +from __future__ import annotations + +import os +import struct +from collections import defaultdict +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO + +_MAX_METADATA_ITEMS = 100_000 +_MAX_TENSORS = 100_000 +_MAX_ARRAY_ITEMS = 2_000_000 +_MAX_ITERATED_VALUES = 500_000 +_MAX_STRING_BYTES = 64 * 1024 * 1024 +_MAX_TOTAL_KEY_BYTES = 4 * 1024 * 1024 +_MAX_DIMENSIONS = 4 +_MAX_PARAMETER_COUNT = 100_000_000_000_000 + +_FIXED_VALUE_FORMATS = { + 0: " dict[str, object]: + return { + "gguf_endianness": "little", + "gguf_version": self.version, + "tensor_count": self.tensor_count, + "header_tensor_element_count": self.header_tensor_element_count, + "gguf_file_type": self.file_type, + "nominal_quantization_bits": self.nominal_quantization_bits, + "split_count": self.split_count, + "tensor_type_parameter_counts": { + str(key): value for key, value in sorted(self.tensor_type_parameter_counts.items()) + }, + "tensor_type_consistent": self.tensor_type_consistent, + } + + +@dataclass(slots=True) +class _WorkBudget: + remaining: int = _MAX_ITERATED_VALUES + + def consume(self, amount: int) -> None: + if amount < 0 or amount > self.remaining: + raise ValueError("GGUF metadata exceeds the parsing-work budget") + self.remaining -= amount + + +def inspect_gguf(path: Path) -> GGUFModelInfo: + """Read bounded metadata and tensor descriptors from one local GGUF.""" + + try: + if path.is_symlink(): + raise ValueError("GGUF artifact cannot be a symbolic link") + with path.open("rb") as handle: + file_size = os.fstat(handle.fileno()).st_size + if _read_exact(handle, 4) != b"GGUF": + raise ValueError("invalid GGUF magic") + version = _u32(handle) + if version not in {2, 3}: + raise ValueError("unsupported GGUF version or byte order") + tensor_count = _u64(handle) + metadata_count = _u64(handle) + if not 0 < tensor_count <= _MAX_TENSORS: + raise ValueError("GGUF tensor count is outside the safety bound") + if metadata_count > _MAX_METADATA_ITEMS: + raise ValueError("GGUF metadata count is outside the safety bound") + + budget = _WorkBudget() + special_values: dict[str, int] = {} + seen_keys: set[str] = set() + total_key_bytes = 0 + for _ in range(metadata_count): + key = _read_string(handle, file_size, maximum=65_535, decode=True) + assert isinstance(key, str) + total_key_bytes += len(key) + if total_key_bytes > _MAX_TOTAL_KEY_BYTES: + raise ValueError("GGUF metadata keys exceed the total byte budget") + if key in seen_keys: + raise ValueError("GGUF metadata keys must be unique") + seen_keys.add(key) + value_type = _u32(handle) + if key in _SPECIAL_INTEGER_KEYS: + if value_type != _SPECIAL_INTEGER_TYPES[key]: + raise ValueError("GGUF integer evidence has an invalid metadata type") + special_values[key] = _read_nonnegative_integer(handle, value_type) + else: + _skip_value(handle, file_size, value_type, depth=0, budget=budget) + + file_type = special_values.get("general.file_type") + split_count = _validate_split_metadata(special_values, tensor_count) + parameter_count = 0 + tensor_type_parameters: defaultdict[int, int] = defaultdict(int) + for _ in range(tensor_count): + _read_string(handle, file_size, maximum=64, decode=False) + dimensions_count = _u32(handle) + if not 0 < dimensions_count <= _MAX_DIMENSIONS: + raise ValueError("GGUF tensor dimension count is outside the safety bound") + elements = 1 + for _ in range(dimensions_count): + dimension = _u64(handle) + if dimension <= 0: + raise ValueError("GGUF tensor dimensions must be positive") + elements *= dimension + if elements > _MAX_PARAMETER_COUNT: + raise ValueError("GGUF tensor parameter count is outside the safety bound") + tensor_type = _u32(handle) + _u64(handle) # tensor-data-relative offset; llama.cpp validates payloads + tensor_type_parameters[tensor_type] += elements + parameter_count += elements + if parameter_count > _MAX_PARAMETER_COUNT: + raise ValueError("GGUF parameter count is outside the safety bound") + except OSError as exc: + raise ValueError("could not inspect GGUF header") from exc + + nominal_bits = nominal_quantization_bits(file_type) + consistency = tensor_type_evidence_consistent( + file_type, + tensor_type_parameters, + parameter_count, + ) + return GGUFModelInfo( + version=version, + tensor_count=tensor_count, + header_tensor_element_count=parameter_count, + file_type=file_type, + nominal_quantization_bits=nominal_bits, + split_count=split_count, + tensor_type_parameter_counts=dict(tensor_type_parameters), + tensor_type_consistent=consistency, + ) + + +def nominal_quantization_bits(file_type: object) -> float | None: + """Return the pinned runtime's nominal majority bit class, if eligible.""" + + if isinstance(file_type, bool) or not isinstance(file_type, int): + return None + return _NOMINAL_QUANTIZATION_BITS.get(file_type) + + +def tensor_type_evidence_consistent( + file_type: object, + parameter_counts: object, + total_parameters: object, +) -> bool | None: + """Recompute whether the declared class dominates raw tensor descriptors.""" + + if isinstance(file_type, bool) or not isinstance(file_type, int): + return None + expected_type = _EXPECTED_MAJORITY_TENSOR_TYPES.get(file_type) + if expected_type is None: + return None + if ( + not isinstance(parameter_counts, Mapping) + or isinstance(total_parameters, bool) + or not isinstance(total_parameters, int) + or total_parameters <= 0 + ): + return False + normalized: dict[int, int] = {} + for raw_type, raw_count in parameter_counts.items(): + if isinstance(raw_type, bool): + return False + if isinstance(raw_type, int): + tensor_type = raw_type + elif isinstance(raw_type, str) and raw_type.isdecimal(): + tensor_type = int(raw_type) + else: + return False + if ( + not 0 <= tensor_type <= 1024 + or tensor_type in normalized + or isinstance(raw_count, bool) + or not isinstance(raw_count, int) + or raw_count < 0 + ): + return False + normalized[tensor_type] = raw_count + if not normalized or sum(normalized.values()) != total_parameters: + return False + return normalized.get(expected_type, 0) * 2 > total_parameters + + +def _validate_split_metadata(values: dict[str, int], tensor_count: int) -> int: + split_keys = {"split.no", "split.count", "split.tensors.count"} + present = split_keys.intersection(values) + if not present: + return 1 + if present != split_keys: + raise ValueError("GGUF split metadata is incomplete") + split_number = values["split.no"] + split_count = values["split.count"] + split_tensors = values["split.tensors.count"] + if split_count != 1 or split_number != 0: + raise ValueError("split GGUF checkpoints are not allowed") + if split_tensors != tensor_count: + raise ValueError("single-file GGUF split tensor count is inconsistent") + return split_count + + +def _read_exact(handle: BinaryIO, size: int) -> bytes: + value = handle.read(size) + if len(value) != size: + raise ValueError("truncated GGUF header") + return value + + +def _u32(handle: BinaryIO) -> int: + return int(struct.unpack(" int: + return int(struct.unpack(" int: + if value_type not in {0, 1, 2, 3, 4, 5, 10, 11}: + raise ValueError("GGUF integer evidence has an invalid metadata type") + value = struct.unpack( + _FIXED_VALUE_FORMATS[value_type], + _read_exact(handle, _FIXED_VALUE_SIZES[value_type]), + )[0] + if not isinstance(value, int) or value < 0: + raise ValueError("GGUF integer evidence must be non-negative") + return value + + +def _read_string( + handle: BinaryIO, + file_size: int, + *, + maximum: int, + decode: bool, +) -> str | None: + length = _u64(handle) + if length > maximum: + raise ValueError("GGUF string exceeds its safety bound") + if decode: + raw = _read_exact(handle, length) + try: + return raw.decode("ascii") + except UnicodeDecodeError as exc: + raise ValueError("GGUF metadata key is not ASCII") from exc + _skip_bytes(handle, file_size, length) + return None + + +def _skip_value( + handle: BinaryIO, + file_size: int, + value_type: int, + *, + depth: int, + budget: _WorkBudget, +) -> None: + fixed_size = _FIXED_VALUE_SIZES.get(value_type) + if fixed_size is not None: + _skip_bytes(handle, file_size, fixed_size) + return + if value_type == 8: + _read_string(handle, file_size, maximum=_MAX_STRING_BYTES, decode=False) + return + if value_type != 9 or depth >= 4: + raise ValueError("unsupported GGUF metadata value type") + element_type = _u32(handle) + if element_type not in {*_FIXED_VALUE_SIZES, 8, 9}: + raise ValueError("unsupported GGUF metadata array element type") + length = _u64(handle) + if length > _MAX_ARRAY_ITEMS: + raise ValueError("GGUF metadata array exceeds its safety bound") + element_size = _FIXED_VALUE_SIZES.get(element_type) + if element_size is not None: + _skip_bytes(handle, file_size, length * element_size) + return + budget.consume(length) + for _ in range(length): + _skip_value( + handle, + file_size, + element_type, + depth=depth + 1, + budget=budget, + ) + + +def _skip_bytes(handle: BinaryIO, file_size: int, size: int) -> None: + if size < 0 or handle.tell() + size > file_size: + raise ValueError("truncated GGUF header") + handle.seek(size, 1) diff --git a/src/fallthrough/local/identity.py b/src/fallthrough/local/identity.py index 451e77e..d70b267 100644 --- a/src/fallthrough/local/identity.py +++ b/src/fallthrough/local/identity.py @@ -10,7 +10,6 @@ from pathlib import Path _EXPLICIT_LABEL_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") -_IMMUTABLE_REVISION_RE = re.compile(r"[0-9a-fA-F]{40}\Z") @dataclass(frozen=True, slots=True) @@ -23,25 +22,10 @@ class CheckpointIdentity: total_bytes: int | None -def identify_checkpoint(model: str, revision: str | None = None) -> CheckpointIdentity: - """Hash a local checkpoint or bind a cached identifier to an immutable revision.""" +def identify_checkpoint(model: str) -> CheckpointIdentity: + """Hash one existing, regular, non-symlink GGUF model artifact.""" - path = Path(model).expanduser() - if path.exists(): - if revision is not None: - raise ValueError("--model-revision cannot be used with an existing local path") - return _hash_local_checkpoint(path) - if revision is None or _IMMUTABLE_REVISION_RE.fullmatch(revision) is None: - raise ValueError( - "cached model identifiers require --model-revision with a full 40-digit hex commit" - ) - digest = hashlib.sha256(f"identifier\0{model}\0{revision.lower()}".encode()).hexdigest() - return CheckpointIdentity( - kind="cached_identifier_revision", - identity_sha256=digest, - file_count=None, - total_bytes=None, - ) + return _hash_local_checkpoint(Path(model).expanduser()) def unverified_checkpoint_identity(model: str) -> CheckpointIdentity: @@ -84,8 +68,14 @@ def experiment_fingerprint( backend: str, model_label: str, checkpoint_identity_sha256: str | None, + checkpoint_kind: str | None = None, + checkpoint_file_count: int | None = None, + checkpoint_total_bytes: int | None = None, configuration: Mapping[str, object], hardware: Mapping[str, object], + implementation_sha256: str | None = None, + shared_implementation_sha256: str | None = None, + benchmark_context_fingerprint: str | None = None, ) -> str: """Hash the material dimensions needed to compare local measurements.""" @@ -96,60 +86,78 @@ def experiment_fingerprint( "hardware": hardware, "model_label": model_label, "checkpoint_identity_sha256": checkpoint_identity_sha256, + "checkpoint_kind": checkpoint_kind, + "checkpoint_file_count": checkpoint_file_count, + "checkpoint_total_bytes": checkpoint_total_bytes, + "implementation_sha256": implementation_sha256, + "execution_mode": "local_shadow", + "shared_implementation_sha256": shared_implementation_sha256, + "benchmark_context_fingerprint": benchmark_context_fingerprint, }, allow_nan=False, ensure_ascii=True, separators=(",", ":"), sort_keys=True, ) - return hashlib.blake2s(payload.encode("utf-8"), digest_size=8).hexdigest() + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def source_tree_fingerprint(*extra_sources: Path) -> str: + """Hash executable Python sources used by an experiment. + + Package version numbers can remain unchanged across a dirty development + tree. Content identity prevents measurements from different implementations + being silently treated as one run while remaining independent of checkout + paths. + """ + + package_root = Path(__file__).resolve().parents[1] + sources: list[tuple[str, Path]] = [ + (f"fallthrough/{path.relative_to(package_root).as_posix()}", path) + for path in package_root.rglob("*.py") + ] + sources.extend( + (f"extra/{index}/{path.name}", path.resolve()) for index, path in enumerate(extra_sources) + ) + digest = hashlib.sha256() + try: + for label, path in sorted(sources, key=lambda item: item[0]): + # Git checkouts may use CRLF on Windows and LF in the Linux grader. + # Source identity is semantic text identity, not checkout EOL style. + content = path.read_bytes().replace(b"\r\n", b"\n") + digest.update(label.encode("utf-8")) + digest.update(b"\0") + digest.update(len(content).to_bytes(8, "big")) + digest.update(content) + except OSError as exc: + raise ValueError(f"could not hash implementation sources ({type(exc).__name__})") from exc + return digest.hexdigest() def _hash_local_checkpoint(path: Path) -> CheckpointIdentity: try: if path.is_symlink(): raise ValueError("local checkpoint paths cannot be symbolic links") - if path.is_file(): - if path.suffix.casefold() != ".safetensors": - raise ValueError("a local model file must use the .safetensors format") - files = [path] - root = path.parent - elif path.is_dir(): - entries = list(path.rglob("*")) - if any(candidate.is_symlink() for candidate in entries): - raise ValueError("local checkpoint directories cannot contain symbolic links") - files = sorted( - (candidate for candidate in entries if candidate.is_file()), - key=lambda candidate: candidate.relative_to(path).as_posix(), - ) - if not files: - raise ValueError("local model directory is empty") - if not any(candidate.suffix.casefold() == ".safetensors" for candidate in files): - raise ValueError("local model directory contains no safetensors weights") - root = path - else: - raise ValueError("local model path is neither a file nor a directory") - + if not path.is_file(): + raise ValueError("local model must be an existing GGUF file") + if path.suffix.casefold() != ".gguf": + raise ValueError("local model file must use the .gguf format") digest = hashlib.sha256() - total_bytes = 0 - resolved_root = root.resolve() - for candidate in files: - if not candidate.resolve().is_relative_to(resolved_root): - raise ValueError("local checkpoint file resolves outside its model directory") - relative = candidate.relative_to(root).as_posix().encode("utf-8") - digest.update(len(relative).to_bytes(8, "big")) - digest.update(relative) - size = candidate.stat().st_size - total_bytes += size - digest.update(size.to_bytes(16, "big")) - with candidate.open("rb") as handle: - while chunk := handle.read(1024 * 1024): - digest.update(chunk) + size = path.stat().st_size + if size <= 0: + raise ValueError("local GGUF model file is empty") + with path.open("rb") as handle: + header = handle.read(4) + if header != b"GGUF": + raise ValueError("local model does not have a GGUF file header") + digest.update(header) + while chunk := handle.read(1024 * 1024): + digest.update(chunk) except OSError as exc: raise ValueError(f"could not hash local checkpoint ({type(exc).__name__})") from exc return CheckpointIdentity( - kind="local_content_sha256", + kind="local_gguf_sha256", identity_sha256=digest.hexdigest(), - file_count=len(files), - total_bytes=total_bytes, + file_count=1, + total_bytes=size, ) diff --git a/src/fallthrough/local/llama_cpp_backend.py b/src/fallthrough/local/llama_cpp_backend.py new file mode 100644 index 0000000..be1a8bf --- /dev/null +++ b/src/fallthrough/local/llama_cpp_backend.py @@ -0,0 +1,742 @@ +"""Owned-process llama.cpp backend for baked, CPU-only GGUF inference. + +The worker loads one model once and communicates through an OS pipe. A request +that exceeds its wall budget terminates the worker, giving the parent a real +cancellation boundary around native llama.cpp code. This module never downloads +models and imports ``llama_cpp`` only inside the worker process. +""" + +from __future__ import annotations + +import asyncio +import importlib +import importlib.metadata +import math +import multiprocessing +import os +import platform +import re +import sys +import threading +import time +from collections.abc import Callable, Mapping +from multiprocessing.connection import Connection +from pathlib import Path +from typing import Any, Protocol + +from .backend import LocalBackendError, LocalGeneration +from .gguf_metadata import GGUFModelInfo, inspect_gguf + +_MAX_GGUF_BYTES = 3 * 1024**3 +_MAX_LOAD_TIMEOUT_SECONDS = 55.0 +_MAX_GENERATION_TIMEOUT_SECONDS = 25.0 +_URL_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://") +_SPLIT_FILENAME_RE = re.compile(r"-\d{5}-of-(\d{5})\.gguf\Z", re.IGNORECASE) + + +class _WorkerSession(Protocol): + ready: Mapping[str, object] + + def request(self, payload: Mapping[str, object], timeout_seconds: float) -> dict[str, object]: + """Send one request or fail after a bounded wall time.""" + + def close(self) -> None: + """Stop the owned worker process.""" + + +class _ConnectionLike(Protocol): + def send(self, obj: object) -> None: ... + + def recv(self) -> Any: ... + + def poll(self, timeout: float = 0.0) -> bool: ... + + def close(self) -> None: ... + + +class _ProcessLike(Protocol): + def is_alive(self) -> bool: ... + + def terminate(self) -> None: ... + + def join(self, timeout: float | None = None) -> None: ... + + +class _ProcessSession: + def __init__( + self, + connection: _ConnectionLike, + process: _ProcessLike, + ready: Mapping[str, object], + ) -> None: + self._connection = connection + self._process = process + self._lock = threading.Lock() + self._failed = False + self.ready = ready + + @classmethod + def start( + cls, + *, + model: str, + threads: int, + context_size: int, + batch_size: int, + load_timeout_seconds: float, + ) -> _ProcessSession: + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe(duplex=True) + process = context.Process( + target=_worker_main, + args=(child, model, threads, context_size, batch_size), + name="fallthrough-llama-cpp", + daemon=True, + ) + process.start() + child.close() + if not parent.poll(load_timeout_seconds): + _stop_process(process) + parent.close() + raise LocalBackendError("llama.cpp model load exceeded its wall-time budget") + try: + ready = parent.recv() + except (EOFError, OSError) as exc: + _stop_process(process) + parent.close() + raise LocalBackendError("llama.cpp worker ended during model load") from exc + if not isinstance(ready, Mapping) or ready.get("status") != "ready": + _stop_process(process) + parent.close() + error_type = ready.get("error_type") if isinstance(ready, Mapping) else None + suffix = f" ({error_type})" if isinstance(error_type, str) else "" + raise LocalBackendError(f"llama.cpp model load failed{suffix}") + return cls(parent, process, dict(ready)) + + def request(self, payload: Mapping[str, object], timeout_seconds: float) -> dict[str, object]: + with self._lock: + started = time.perf_counter() + if self._failed or not self._process.is_alive(): + raise LocalBackendError("llama.cpp worker is unavailable") + try: + self._connection.send(dict(payload)) + except (BrokenPipeError, EOFError, OSError) as exc: + self._fail() + raise LocalBackendError("could not send work to llama.cpp") from exc + if not self._connection.poll(timeout_seconds): + peak_memory, memory_measurement = _peak_memory_bytes() + self._fail() + raise LocalBackendError( + "llama.cpp generation exceeded its wall-time budget", + latency_ms=(time.perf_counter() - started) * 1000.0, + peak_memory_bytes=peak_memory, + memory_measurement=memory_measurement, + ) + try: + response = self._connection.recv() + except (EOFError, OSError) as exc: + self._fail() + raise LocalBackendError("llama.cpp worker ended during generation") from exc + if not isinstance(response, dict): + self._fail() + raise LocalBackendError("llama.cpp worker returned an invalid response") + return response + + def close(self) -> None: + with self._lock: + if not self._failed and self._process.is_alive(): + try: + self._connection.send({"operation": "close"}) + if self._connection.poll(2.0): + self._connection.recv() + except (BrokenPipeError, EOFError, OSError): + pass + _stop_process(self._process) + try: + self._connection.close() + except OSError: + pass + self._failed = True + + def _fail(self) -> None: + self._failed = True + _stop_process(self._process) + try: + self._connection.close() + except OSError: + pass + + +class LlamaCppBackend: + """Serialized local GGUF generation with an owned worker process.""" + + backend_name = "llama_cpp" + + def __init__( + self, + *, + model: str, + session: _WorkerSession, + artifact_info: GGUFModelInfo, + load_timeout_seconds: float, + generation_timeout_seconds: float, + ) -> None: + ready = session.ready + self.model = model + self._session = session + self._generation_timeout_seconds = generation_timeout_seconds + self._async_lock = asyncio.Lock() + self.load_time_seconds = _required_number(ready, "load_time_seconds") + self.load_peak_memory_bytes = _optional_nonnegative_int(ready.get("peak_memory_bytes")) + self.configuration = _mapping(ready.get("configuration")) + runtime_parameter_count = _required_nonnegative_int( + self.configuration, + "runtime_parameter_count", + ) + runtime_file_type = _required_nonnegative_int( + self.configuration, + "runtime_file_type", + ) + if runtime_parameter_count <= 0: + raise LocalBackendError("llama.cpp worker returned an empty model") + if runtime_parameter_count != artifact_info.header_tensor_element_count: + raise LocalBackendError("GGUF header and loaded runtime parameter counts disagree") + if artifact_info.file_type is not None and runtime_file_type != artifact_info.file_type: + raise LocalBackendError("GGUF header and loaded runtime file types disagree") + self.configuration["load_timeout_seconds"] = load_timeout_seconds + self.configuration["generation_timeout_seconds"] = generation_timeout_seconds + self.configuration.update(artifact_info.as_configuration()) + self.configuration["parameter_count"] = runtime_parameter_count + self.configuration["parameter_count_crosscheck"] = "match" + self.configuration["file_type_crosscheck"] = ( + "match" if artifact_info.file_type is not None else "header_missing" + ) + self.configuration["quantization_evidence"] = ( + "general.file_type_nominal_majority_crosschecked_runtime" + if artifact_info.nominal_quantization_bits is not None + else None + ) + self.hardware = _mapping(ready.get("hardware")) + measurement = ready.get("memory_measurement") + self.memory_measurement = ( + measurement if isinstance(measurement, str) else "worker_process_peak_rss_bytes" + ) + + @classmethod + def load( + cls, + model: str, + *, + threads: int = 2, + context_size: int = 1024, + batch_size: int = 128, + load_timeout_seconds: float = 50.0, + generation_timeout_seconds: float = 25.0, + session_factory: Callable[..., _WorkerSession] | None = None, + ) -> LlamaCppBackend: + model_path = _validate_load_options( + model, + threads, + context_size, + batch_size, + load_timeout_seconds, + generation_timeout_seconds, + ) + artifact_info = inspect_gguf(model_path) + factory = session_factory or _ProcessSession.start + session = factory( + model=str(model_path), + threads=threads, + context_size=context_size, + batch_size=batch_size, + load_timeout_seconds=load_timeout_seconds, + ) + try: + return cls( + model=str(model_path), + session=session, + artifact_info=artifact_info, + load_timeout_seconds=load_timeout_seconds, + generation_timeout_seconds=generation_timeout_seconds, + ) + except Exception: + session.close() + raise + + def generate_sync( + self, + prompt: str, + *, + max_tokens: int, + temperature: float = 0.0, + ) -> LocalGeneration: + _validate_generation_options(prompt, max_tokens, temperature) + response = self._session.request( + { + "operation": "generate", + "prompt": prompt, + "max_tokens": max_tokens, + "temperature": float(temperature), + }, + self._generation_timeout_seconds, + ) + if response.get("status") != "ok": + error_type = response.get("error_type") + suffix = f" ({error_type})" if isinstance(error_type, str) else "" + raw_memory_measurement = response.get("memory_measurement") + error_memory_measurement = ( + raw_memory_measurement + if isinstance(raw_memory_measurement, str) + else self.memory_measurement + ) + raise LocalBackendError( + f"llama.cpp generation failed{suffix}", + latency_ms=_optional_number(response.get("latency_ms")), + peak_memory_bytes=_optional_nonnegative_int(response.get("peak_memory_bytes")), + memory_measurement=error_memory_measurement, + ) + text = response.get("text") + if not isinstance(text, str): + raise LocalBackendError("llama.cpp completion text is not a string") + token_count = _required_nonnegative_int(response, "completion_tokens") + prompt_tokens = _optional_nonnegative_int(response.get("prompt_tokens")) + latency_ms = _required_number(response, "latency_ms") + peak_memory = _optional_nonnegative_int(response.get("peak_memory_bytes")) + measurement = response.get("memory_measurement") + finish_reason = response.get("finish_reason") + cumulative_logprob = response.get("cumulative_logprob") + if not isinstance(cumulative_logprob, int | float) or not math.isfinite(cumulative_logprob): + cumulative_logprob = None + return LocalGeneration( + text=text, + token_count=token_count, + latency_ms=latency_ms, + prompt_token_count=prompt_tokens, + peak_memory_bytes=peak_memory, + finish_reason=finish_reason if isinstance(finish_reason, str) else None, + metadata={ + "cumulative_logprob": cumulative_logprob, + "memory_measurement": ( + measurement if isinstance(measurement, str) else self.memory_measurement + ), + "runtime_version": self.configuration.get("llama_cpp_python_version"), + }, + ) + + async def generate( + self, + prompt: str, + *, + max_tokens: int, + temperature: float = 0.0, + ) -> LocalGeneration: + async with self._async_lock: + return await asyncio.to_thread( + self.generate_sync, + prompt, + max_tokens=max_tokens, + temperature=temperature, + ) + + def close(self) -> None: + self._session.close() + + +def _worker_main( + connection: Connection, + model: str, + threads: int, + context_size: int, + batch_size: int, +) -> None: + engine: Any | None = None + try: + started = time.perf_counter() + try: + llama_cpp = importlib.import_module("llama_cpp") + engine = llama_cpp.Llama( + model_path=model, + n_ctx=context_size, + n_batch=batch_size, + n_threads=threads, + n_threads_batch=threads, + n_gpu_layers=0, + seed=0, + use_mmap=True, + use_mlock=False, + logits_all=False, + embedding=False, + verbose=False, + ) + runtime_evidence = _runtime_model_evidence(llama_cpp, engine) + except Exception as exc: + connection.send({"status": "error", "error_type": type(exc).__name__}) + return + version = getattr(llama_cpp, "__version__", None) + if not isinstance(version, str): + try: + version = importlib.metadata.version("llama-cpp-python") + except importlib.metadata.PackageNotFoundError: + version = None + configuration: dict[str, object] = { + "batch_size": batch_size, + "chat_template_source": "gguf_metadata", + "context_size": context_size, + "llama_cpp_python_version": version, + "n_gpu_layers": 0, + "seed": 0, + "threads": threads, + "use_mlock": False, + "use_mmap": True, + **runtime_evidence, + } + hardware: dict[str, object] = { + "cpu_count_visible": os.cpu_count(), + "machine": platform.machine(), + "threads_used": threads, + } + memory_limit = _cgroup_memory_limit_bytes() + if memory_limit is not None: + hardware["cgroup_memory_limit_bytes"] = memory_limit + peak_memory, memory_measurement = _peak_memory_bytes() + connection.send( + { + "status": "ready", + "load_time_seconds": time.perf_counter() - started, + "peak_memory_bytes": peak_memory, + "memory_measurement": memory_measurement, + "configuration": configuration, + "hardware": hardware, + } + ) + while True: + try: + request = connection.recv() + except EOFError: + break + if not isinstance(request, Mapping): + connection.send({"status": "error", "error_type": "InvalidRequest"}) + continue + operation = request.get("operation") + if operation == "close": + connection.send({"status": "closed"}) + break + if operation != "generate": + connection.send({"status": "error", "error_type": "InvalidOperation"}) + continue + connection.send(_worker_generate(engine, request)) + finally: + if engine is not None: + close = getattr(engine, "close", None) + if callable(close): + try: + close() + except Exception: + pass + try: + connection.close() + except OSError: + pass + + +def _runtime_model_evidence(module: Any, engine: Any) -> dict[str, int]: + model = getattr(engine, "model", None) + if model is None: + raise RuntimeError("loaded llama.cpp engine did not expose its model") + evidence: dict[str, int] = {} + for output_key, function_name in ( + ("runtime_parameter_count", "llama_model_n_params"), + ("runtime_file_type", "llama_model_ftype"), + ("runtime_tensor_bytes", "llama_model_size"), + ): + function = getattr(module, function_name, None) + if not callable(function): + raise RuntimeError("llama.cpp runtime evidence API is unavailable") + value = function(model) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError("llama.cpp runtime evidence is invalid") + evidence[output_key] = value + if evidence["runtime_parameter_count"] <= 0 or evidence["runtime_tensor_bytes"] <= 0: + raise RuntimeError("llama.cpp loaded an empty model") + return evidence + + +def _worker_generate(engine: Any, request: Mapping[str, object]) -> dict[str, object]: + started = time.perf_counter() + try: + prompt = request["prompt"] + max_tokens = request["max_tokens"] + temperature = request["temperature"] + if not isinstance(prompt, str): + raise TypeError("prompt") + if not isinstance(max_tokens, int) or isinstance(max_tokens, bool): + raise TypeError("max_tokens") + if not isinstance(temperature, int | float) or isinstance(temperature, bool): + raise TypeError("temperature") + response = engine.create_chat_completion( + messages=[{"role": "user", "content": prompt}], + max_tokens=max_tokens, + temperature=float(temperature), + top_p=1.0, + top_k=1, + seed=0, + stream=False, + ) + text, finish_reason, cumulative_logprob = _parse_completion(response) + usage = response.get("usage") if isinstance(response, Mapping) else None + prompt_tokens = _usage_token(usage, "prompt_tokens") + completion_tokens = _usage_token(usage, "completion_tokens") + if completion_tokens is None: + completion_tokens = _count_tokens(engine, text, add_bos=False) + peak_memory, memory_measurement = _peak_memory_bytes() + return { + "status": "ok", + "text": text, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "finish_reason": finish_reason, + "cumulative_logprob": cumulative_logprob, + "latency_ms": (time.perf_counter() - started) * 1000.0, + "peak_memory_bytes": peak_memory, + "memory_measurement": memory_measurement, + } + except Exception as exc: + peak_memory, memory_measurement = _peak_memory_bytes() + return { + "status": "error", + "error_type": type(exc).__name__, + "latency_ms": (time.perf_counter() - started) * 1000.0, + "peak_memory_bytes": peak_memory, + "memory_measurement": memory_measurement, + } + + +def _parse_completion(response: object) -> tuple[str, str | None, float | None]: + if not isinstance(response, Mapping): + raise TypeError("response") + choices = response.get("choices") + if not isinstance(choices, list) or not choices or not isinstance(choices[0], Mapping): + raise ValueError("choices") + choice = choices[0] + message = choice.get("message") + text: object = None + if isinstance(message, Mapping): + text = message.get("content") + if text is None: + text = choice.get("text") + if not isinstance(text, str): + raise TypeError("text") + finish_reason = choice.get("finish_reason") + cumulative = _completion_logprob(choice.get("logprobs")) + return text, finish_reason if isinstance(finish_reason, str) else None, cumulative + + +def _completion_logprob(logprobs: object) -> float | None: + if not isinstance(logprobs, Mapping): + return None + values: list[float] = [] + content = logprobs.get("content") + if isinstance(content, list): + for item in content: + if isinstance(item, Mapping): + value = item.get("logprob") + if isinstance(value, int | float) and math.isfinite(value): + values.append(float(value)) + legacy = logprobs.get("token_logprobs") + if not values and isinstance(legacy, list): + for value in legacy: + if isinstance(value, int | float) and math.isfinite(value): + values.append(float(value)) + return sum(values) if values else None + + +def _usage_token(usage: object, key: str) -> int | None: + if not isinstance(usage, Mapping): + return None + return _optional_nonnegative_int(usage.get(key)) + + +def _count_tokens(engine: Any, text: str, *, add_bos: bool) -> int: + tokens = engine.tokenize(text.encode("utf-8"), add_bos=add_bos, special=True) + if not hasattr(tokens, "__len__"): + raise TypeError("tokens") + return len(tokens) + + +def _validate_load_options( + model: str, + threads: int, + context_size: int, + batch_size: int, + load_timeout_seconds: float, + generation_timeout_seconds: float, +) -> Path: + if not isinstance(model, str) or not model.strip(): + raise ValueError("model must be a nonblank local GGUF path") + if _URL_RE.match(model): + raise ValueError("model must be a local GGUF path, not a URL") + path = Path(model).expanduser() + if path.is_symlink() or not path.is_file() or path.suffix.casefold() != ".gguf": + raise ValueError("model must be an existing, regular, non-symlink .gguf file") + split_match = _SPLIT_FILENAME_RE.search(path.name) + if split_match is not None and int(split_match.group(1)) > 1: + raise ValueError("split GGUF checkpoints are not allowed") + size = path.stat().st_size + if size <= 4 or size > _MAX_GGUF_BYTES: + raise ValueError("GGUF model size is outside the final-runtime safety bound") + with path.open("rb") as handle: + if handle.read(4) != b"GGUF": + raise ValueError("model does not have a GGUF file header") + if isinstance(threads, bool) or not 1 <= threads <= 2: + raise ValueError("threads must be 1 or 2 for the final grader") + if isinstance(context_size, bool) or not 128 <= context_size <= 4096: + raise ValueError("context_size must be between 128 and 4096") + if isinstance(batch_size, bool) or not 1 <= batch_size <= min(context_size, 512): + raise ValueError("batch_size must be positive and no larger than context_size or 512") + for name, value, maximum in ( + ("load_timeout_seconds", load_timeout_seconds, _MAX_LOAD_TIMEOUT_SECONDS), + ( + "generation_timeout_seconds", + generation_timeout_seconds, + _MAX_GENERATION_TIMEOUT_SECONDS, + ), + ): + if ( + isinstance(value, bool) + or not isinstance(value, int | float) + or not math.isfinite(value) + or not 0.0 < value <= maximum + ): + raise ValueError(f"{name} must be finite, positive, and no greater than {maximum}") + return path.resolve(strict=True) + + +def _validate_generation_options(prompt: str, max_tokens: int, temperature: float) -> None: + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError("prompt must be nonblank text") + if isinstance(max_tokens, bool) or not isinstance(max_tokens, int) or max_tokens <= 0: + raise ValueError("max_tokens must be a positive integer") + if ( + isinstance(temperature, bool) + or not isinstance(temperature, int | float) + or not math.isfinite(temperature) + or not 0.0 <= temperature <= 2.0 + ): + raise ValueError("temperature must be finite and between 0 and 2") + + +def _required_number(mapping: Mapping[str, object], key: str) -> float: + value = mapping.get(key) + if ( + isinstance(value, bool) + or not isinstance(value, int | float) + or not math.isfinite(value) + or value < 0 + ): + raise LocalBackendError(f"llama.cpp worker returned invalid {key}") + return float(value) + + +def _optional_number(value: object) -> float | None: + if ( + isinstance(value, bool) + or not isinstance(value, int | float) + or not math.isfinite(value) + or value < 0 + ): + return None + return float(value) + + +def _required_nonnegative_int(mapping: Mapping[str, object], key: str) -> int: + value = _optional_nonnegative_int(mapping.get(key)) + if value is None: + raise LocalBackendError(f"llama.cpp worker returned invalid {key}") + return value + + +def _optional_nonnegative_int(value: object) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise LocalBackendError("llama.cpp worker returned an invalid integer metric") + return value + + +def _mapping(value: object) -> dict[str, object]: + if not isinstance(value, Mapping): + raise LocalBackendError("llama.cpp worker returned invalid metadata") + return {str(key): item for key, item in value.items()} + + +def _peak_rss_bytes() -> int | None: + if sys.platform.startswith("linux"): + try: + for line in Path("/proc/self/status").read_text(encoding="utf-8").splitlines(): + if line.startswith("VmHWM:"): + parts = line.split() + if len(parts) >= 2: + return int(parts[1]) * 1024 + except (OSError, ValueError): + pass + try: + resource = importlib.import_module("resource") + peak = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + except Exception: + return None + return peak if sys.platform == "darwin" else peak * 1024 + + +def _peak_memory_bytes() -> tuple[int | None, str]: + cgroup_peak = _cgroup_peak_memory_bytes() + if cgroup_peak is not None: + return cgroup_peak, "container_cgroup_peak_memory_bytes" + return _peak_rss_bytes(), "worker_process_peak_rss_bytes" + + +def _cgroup_peak_memory_bytes() -> int | None: + for path in ( + Path("/sys/fs/cgroup/memory.peak"), + Path("/sys/fs/cgroup/memory/memory.max_usage_in_bytes"), + ): + try: + raw = path.read_text(encoding="ascii").strip() + value = int(raw) + except (OSError, ValueError): + continue + if value >= 0: + return value + return None + + +def _cgroup_memory_limit_bytes() -> int | None: + for path in ( + Path("/sys/fs/cgroup/memory.max"), + Path("/sys/fs/cgroup/memory/memory.limit_in_bytes"), + ): + try: + raw = path.read_text(encoding="ascii").strip() + except OSError: + continue + if raw == "max": + return None + try: + value = int(raw) + except ValueError: + continue + if value > 0: + return value + return None + + +def _stop_process(process: _ProcessLike) -> None: + if not process.is_alive(): + process.join(timeout=0.1) + return + process.terminate() + process.join(timeout=1.0) + if process.is_alive(): + kill = getattr(process, "kill", None) + if callable(kill): + kill() + process.join(timeout=1.0) diff --git a/src/fallthrough/local/vllm_backend.py b/src/fallthrough/local/vllm_backend.py deleted file mode 100644 index c6c4bae..0000000 --- a/src/fallthrough/local/vllm_backend.py +++ /dev/null @@ -1,389 +0,0 @@ -"""Optional in-process vLLM backend for pod experiments. - -This module deliberately imports neither vLLM nor PyTorch at module import -time. A normal Fallthrough install therefore has no local-model dependency or -startup cost when ``FALLTHROUGH_LOCAL_BACKEND=none``. -""" - -from __future__ import annotations - -import asyncio -import importlib -import importlib.metadata -import math -import os -import re -import threading -import time -from collections.abc import Iterator -from contextlib import contextmanager -from typing import Any - -from .backend import LocalBackendError, LocalGeneration - - -class _VramMeter: - def __init__(self, torch_module: Any | None) -> None: - self._torch = torch_module - self._observed_global_peak = 0 - - @property - def available(self) -> bool: - try: - return bool(self._torch is not None and self._torch.cuda.is_available()) - except Exception: - return False - - def reset(self) -> None: - self._observed_global_peak = 0 - if not self.available: - return - torch = self._torch - if torch is None: - return - try: - torch.cuda.synchronize() - torch.cuda.reset_peak_memory_stats() - except Exception: - pass - - def peak_bytes(self) -> int | None: - if not self.available: - return None - torch = self._torch - if torch is None: - return None - measurements: list[int] = [] - try: - torch.cuda.synchronize() - except Exception: - pass - try: - measurements.append(int(torch.cuda.max_memory_allocated())) - except Exception: - pass - try: - free, total = torch.cuda.mem_get_info() - measurements.append(max(0, int(total) - int(free))) - except Exception: - pass - if self._observed_global_peak: - measurements.append(self._observed_global_peak) - return max(measurements) if measurements else None - - @contextmanager - def track(self) -> Iterator[None]: - """Poll global device use so child-engine allocations remain observable.""" - - self.reset() - stop = threading.Event() - worker: threading.Thread | None = None - if self.available: - worker = threading.Thread(target=self._poll, args=(stop,), daemon=True) - worker.start() - try: - yield - finally: - stop.set() - if worker is not None: - worker.join(timeout=1.0) - self._sample_global_usage() - - def _poll(self, stop: threading.Event) -> None: - while not stop.is_set(): - self._sample_global_usage() - stop.wait(0.02) - - def _sample_global_usage(self) -> None: - torch = self._torch - if torch is None or not self.available: - return - try: - free, total = torch.cuda.mem_get_info() - used = max(0, int(total) - int(free)) - except Exception: - return - self._observed_global_peak = max(self._observed_global_peak, used) - - -class VllmBackend: - """Single-engine, serialized offline vLLM generation backend.""" - - backend_name = "vllm" - vram_measurement = "observed_global_device_0_peak_bytes" - - def __init__( - self, - *, - model: str, - engine: Any, - sampling_params_type: Any, - torch_module: Any | None, - load_time_seconds: float, - load_peak_vram_bytes: int | None, - vllm_version: str | None, - configuration: dict[str, object], - hardware: dict[str, object], - ) -> None: - self.model = model - self._engine = engine - self._sampling_params_type = sampling_params_type - self._vram = _VramMeter(torch_module) - self._lock = asyncio.Lock() - self.load_time_seconds = load_time_seconds - self.load_peak_vram_bytes = load_peak_vram_bytes - self.vllm_version = vllm_version - self.configuration = configuration - self.hardware = hardware - - @classmethod - def load( - cls, - model: str, - *, - tensor_parallel_size: int = 1, - dtype: str = "auto", - gpu_memory_utilization: float = 0.82, - max_model_len: int | None = 2048, - enforce_eager: bool = False, - download_dir: str | None = None, - revision: str | None = None, - vllm_module: Any | None = None, - torch_module: Any | None = None, - ) -> VllmBackend: - """Load one model into the current process. - - ``vllm_module`` and ``torch_module`` are injectable solely for offline - unit tests; normal callers leave them unset. - """ - - _validate_load_options(model, tensor_parallel_size, gpu_memory_utilization, max_model_len) - if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*://", model): - raise ValueError("model must be a local path or registry identifier, not a URL") - if revision is not None and (not isinstance(revision, str) or not revision.strip()): - raise ValueError("revision must be nonblank when provided") - os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1" - os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1" - os.environ["HF_HUB_OFFLINE"] = "1" - os.environ["TRANSFORMERS_OFFLINE"] = "1" - os.environ["VLLM_NO_USAGE_STATS"] = "1" - os.environ["DO_NOT_TRACK"] = "1" - try: - vllm = vllm_module or importlib.import_module("vllm") - except Exception as exc: - raise LocalBackendError( - "vLLM is unavailable; use FALLTHROUGH_LOCAL_BACKEND=none or the AMD pod runtime" - ) from exc - if torch_module is None: - try: - torch_module = importlib.import_module("torch") - except Exception: - torch_module = None - - meter = _VramMeter(torch_module) - options: dict[str, Any] = { - "model": model, - "tensor_parallel_size": tensor_parallel_size, - "dtype": dtype, - "gpu_memory_utilization": gpu_memory_utilization, - "enforce_eager": enforce_eager, - "trust_remote_code": False, - "load_format": "safetensors", - "runner": "generate", - "generation_config": "vllm", - "seed": 0, - "hf_token": None, - "allowed_local_media_path": "", - } - if max_model_len is not None: - options["max_model_len"] = max_model_len - if download_dir is not None: - options["download_dir"] = download_dir - if revision is not None: - options["revision"] = revision - - started = time.perf_counter() - try: - with meter.track(): - engine = vllm.LLM(**options) - except Exception as exc: - elapsed = time.perf_counter() - started - raise LocalBackendError( - f"vLLM model load failed after {elapsed:.3f}s ({type(exc).__name__})" - ) from exc - load_time = time.perf_counter() - started - version = getattr(vllm, "__version__", None) - if not isinstance(version, str): - try: - version = importlib.metadata.version("vllm") - except importlib.metadata.PackageNotFoundError: - version = None - return cls( - model=model, - engine=engine, - sampling_params_type=vllm.SamplingParams, - torch_module=torch_module, - load_time_seconds=load_time, - load_peak_vram_bytes=meter.peak_bytes(), - vllm_version=version, - configuration={ - "dtype": dtype, - "enforce_eager": enforce_eager, - "generation_config": "vllm", - "gpu_memory_utilization": gpu_memory_utilization, - "load_format": "safetensors", - "max_model_len": max_model_len, - "revision_bound": revision is not None, - "tensor_parallel_size": tensor_parallel_size, - "vllm_version": version, - }, - hardware=_hardware_report(torch_module), - ) - - def generate_sync( - self, - prompt: str, - *, - max_tokens: int, - temperature: float = 0.0, - ) -> LocalGeneration: - if not isinstance(prompt, str) or not prompt.strip(): - raise ValueError("prompt must be nonblank text") - if isinstance(max_tokens, bool) or max_tokens <= 0: - raise ValueError("max_tokens must be positive") - if ( - isinstance(temperature, bool) - or not isinstance(temperature, int | float) - or not math.isfinite(temperature) - or temperature < 0 - ): - raise ValueError("temperature must be a finite non-negative number") - - sampling = self._sampling_params_type( - max_tokens=max_tokens, - temperature=temperature, - seed=0, - logprobs=1, - ) - started = time.perf_counter() - try: - with self._vram.track(): - requests = self._engine.generate([prompt], sampling, use_tqdm=False) - except Exception as exc: - elapsed_ms = (time.perf_counter() - started) * 1000.0 - raise LocalBackendError( - f"vLLM generation failed after {elapsed_ms:.3f}ms ({type(exc).__name__})" - ) from exc - latency_ms = (time.perf_counter() - started) * 1000.0 - try: - request = requests[0] - completion = request.outputs[0] - text = completion.text - except (AttributeError, IndexError, TypeError) as exc: - raise LocalBackendError("vLLM returned no completion") from exc - if not isinstance(text, str): - raise LocalBackendError("vLLM completion text is not a string") - token_ids = getattr(completion, "token_ids", None) - if token_ids is None: - raise LocalBackendError("vLLM completion is missing generated token IDs") - try: - token_count = len(token_ids) - except TypeError as exc: - raise LocalBackendError("vLLM generated token IDs are not sized") from exc - prompt_token_ids = getattr(request, "prompt_token_ids", None) - prompt_token_count = None - if prompt_token_ids is not None: - try: - prompt_token_count = len(prompt_token_ids) - except TypeError: - prompt_token_count = None - finish_reason = getattr(completion, "finish_reason", None) - cumulative_logprob = getattr(completion, "cumulative_logprob", None) - return LocalGeneration( - text=text, - token_count=token_count, - latency_ms=latency_ms, - prompt_token_count=prompt_token_count, - peak_vram_bytes=self._vram.peak_bytes(), - finish_reason=finish_reason if isinstance(finish_reason, str) else None, - metadata={ - "vllm_version": self.vllm_version, - "vram_measurement": self.vram_measurement, - "cumulative_logprob": ( - float(cumulative_logprob) - if isinstance(cumulative_logprob, int | float) - else None - ), - }, - ) - - async def generate( - self, - prompt: str, - *, - max_tokens: int, - temperature: float = 0.0, - ) -> LocalGeneration: - async with self._lock: - return await asyncio.to_thread( - self.generate_sync, - prompt, - max_tokens=max_tokens, - temperature=temperature, - ) - - def close(self) -> None: - """Best-effort shutdown across vLLM versions.""" - - for candidate in (self._engine, getattr(self._engine, "llm_engine", None)): - shutdown = getattr(candidate, "shutdown", None) - if callable(shutdown): - try: - shutdown() - except Exception: - continue - return - - -def _validate_load_options( - model: str, - tensor_parallel_size: int, - gpu_memory_utilization: float, - max_model_len: int | None, -) -> None: - if not isinstance(model, str) or not model.strip(): - raise ValueError("model must be a nonblank path or identifier") - if tensor_parallel_size != 1 or isinstance(tensor_parallel_size, bool): - raise ValueError("tensor_parallel_size must be 1 for measured local experiments") - if ( - isinstance(gpu_memory_utilization, bool) - or not isinstance(gpu_memory_utilization, int | float) - or not math.isfinite(gpu_memory_utilization) - or not 0.0 < gpu_memory_utilization <= 1.0 - ): - raise ValueError("gpu_memory_utilization must be in (0, 1]") - if max_model_len is not None and (isinstance(max_model_len, bool) or max_model_len <= 0): - raise ValueError("max_model_len must be positive") - - -def _hardware_report(torch_module: Any | None) -> dict[str, object]: - """Return only stable, non-secret hardware dimensions used by the run.""" - - report: dict[str, object] = {"gpu_available": False} - if torch_module is None: - return report - try: - if not torch_module.cuda.is_available(): - return report - report["gpu_available"] = True - report["gpu_count"] = int(torch_module.cuda.device_count()) - properties = torch_module.cuda.get_device_properties(0) - except Exception: - return report - architecture = getattr(properties, "gcnArchName", None) - if isinstance(architecture, str) and architecture: - report["gpu_architecture"] = architecture.split(":", 1)[0] - total_memory = getattr(properties, "total_memory", None) - if isinstance(total_memory, int) and not isinstance(total_memory, bool) and total_memory >= 0: - report["total_vram_bytes"] = total_memory - return report diff --git a/src/fallthrough/main.py b/src/fallthrough/main.py index 496037d..98513cf 100644 --- a/src/fallthrough/main.py +++ b/src/fallthrough/main.py @@ -12,6 +12,7 @@ from fallthrough import __version__ from fallthrough.config import ConfigurationError, Settings from fallthrough.fireworks.client import FireworksClient +from fallthrough.policy.evidence import PackagedPolicyEvidence from fallthrough.policy.route_table import RouteTable from fallthrough.policy.router import BaselineRouter from fallthrough.routes.deterministic import DeterministicRoute @@ -104,15 +105,21 @@ async def run(settings: Settings, adapter: EvaluationAdapter) -> int: return 2 try: - route_table = ( - RouteTable.from_json(settings.route_table_path) - if settings.policy == "hybrid-baseline" and settings.route_table_path is not None - else RouteTable.baseline() - ) + packaged_policy_fallback = False + if settings.policy == "hybrid-baseline": + route_table, packaged_policy_fallback = _load_hybrid_route_table(settings) + else: + route_table = RouteTable.baseline(_baseline_remote_route(settings)) except (OSError, ValueError) as exc: _write_failure_results(adapter, tasks, settings.failure_answer) print(f"fallthrough: route-table configuration error: {exc}", file=sys.stderr) return 2 + if packaged_policy_fallback: + print( + "fallthrough: packaged measured policy is incompatible with ALLOWED_MODELS; " + "using baseline-unmeasured without substituting or filtering routes", + file=sys.stderr, + ) remote_required, gemma_required = _remote_requirements(settings.policy, route_table) if remote_required: @@ -133,7 +140,7 @@ async def run(settings: Settings, adapter: EvaluationAdapter) -> int: client = FireworksClient( settings.fireworks_base_url, settings.fireworks_api_key, - settings.allowed_models if settings.allowed_models_configured else None, + settings.allowed_models, connect_timeout_s=settings.connect_timeout_seconds, read_timeout_s=settings.request_timeout_seconds, max_retries=settings.transport_retries, @@ -146,9 +153,10 @@ async def run(settings: Settings, adapter: EvaluationAdapter) -> int: format_retries=settings.format_retries, ) ) + # Register an exact, evidence-bindable route for every allowlisted + # model, including the Gemma also exposed through the unmeasured + # convenience alias above. for model in settings.allowed_models: - if model == settings.gemma_model: - continue routes.register( FireworksModelRoute( client, @@ -172,6 +180,7 @@ async def run(settings: Settings, adapter: EvaluationAdapter) -> int: router=BaselineRouter(policy=settings.policy, route_table=route_table), trace_sink=JsonlTraceSink(settings.trace_path), global_timeout_seconds=settings.global_timeout_seconds, + per_task_timeout_seconds=settings.task_timeout_seconds, per_route_timeout_seconds=settings.request_timeout_seconds, max_concurrency=settings.max_concurrency, failure_answer=settings.failure_answer, @@ -184,11 +193,10 @@ async def run(settings: Settings, adapter: EvaluationAdapter) -> int: adapter.write_results(results) output_written = True except Exception as exc: - fallback_written = output_written if not output_written: - fallback_written = _write_failure_results(adapter, tasks, settings.failure_answer) + _write_failure_results(adapter, tasks, settings.failure_answer) print(f"fallthrough: runtime error: {exc}", file=sys.stderr) - return 0 if fallback_written else 2 + return 2 finally: if client is not None: try: @@ -213,6 +221,31 @@ def _validate_route_table(route_table: RouteTable, routes: RouteRegistry) -> Non ) +def _load_hybrid_route_table(settings: Settings) -> tuple[RouteTable, bool]: + """Load an explicit or image-packaged hybrid table. + + Explicit tables remain strict configuration. A packaged table first has + its evidence validated fail-closed, then activates only when all exact + measured model IDs are present in the injected allowlist. Incompatibility + discards the whole table; evidence is never rebound to another model and + individual routes are never filtered. + """ + + if settings.route_table_path is not None: + return RouteTable.from_json(settings.route_table_path), False + if settings.packaged_route_table_path is None: + return RouteTable.baseline(_baseline_remote_route(settings)), False + if settings.packaged_policy_evidence_path is None: + raise ConfigurationError("packaged route table requires a policy evidence sidecar") + + route_table = RouteTable.from_json(settings.packaged_route_table_path) + PackagedPolicyEvidence.from_json(settings.packaged_policy_evidence_path, route_table) + measured_models = route_table.exact_fireworks_model_ids() + if not measured_models.issubset(settings.allowed_models): + return RouteTable.baseline(_baseline_remote_route(settings)), True + return route_table, False + + def _remote_requirements(policy: str, route_table: RouteTable) -> tuple[bool, bool]: if policy == "deterministic-only": return False, False @@ -222,6 +255,21 @@ def _remote_requirements(policy: str, route_table: RouteTable) -> tuple[bool, bo return any(name.startswith("fw_") for name in route_names), "fw_gemma" in route_names +def _baseline_remote_route(settings: Settings) -> str: + """Prefer Gemma, otherwise select only from the injected allowlist. + + The sentinel keeps missing remote configuration fail-closed without making + Gemma an unavailable hard requirement. It can never survive route-table + validation after valid remote configuration is supplied. + """ + + if settings.gemma_model is not None: + return "fw_gemma" + if settings.allowed_models: + return fireworks_route_name(settings.allowed_models[0]) + return "fw_unconfigured" + + def _write_failure_results( adapter: EvaluationAdapter, tasks: Sequence[Task], diff --git a/src/fallthrough/policy/evidence.py b/src/fallthrough/policy/evidence.py new file mode 100644 index 0000000..8095850 --- /dev/null +++ b/src/fallthrough/policy/evidence.py @@ -0,0 +1,216 @@ +"""Fail-closed evidence binding for an image-packaged measured policy.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from fallthrough.inference import NORMALIZATION_POLICY, PROMPT_POLICY +from fallthrough.local.identity import source_tree_fingerprint +from fallthrough.policy.route_table import RouteTable + +_SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +_MAX_EVIDENCE_BYTES = 256 * 1024 +_ACCURACY_SEMANTICS = { + "official_evaluator", + "transparent_local_proxy_not_official_llm_judge", +} +_TOP_LEVEL_KEYS = { + "schema_version", + "route_table_sha256", + "runtime_implementation_sha256", + "prompt_policy", + "normalization_policy", + "model_ids", + "model_identity_verified", + "dataset_fingerprints", + "run_fingerprints", + "accuracy_evidence", + "token_evidence", +} + + +@dataclass(frozen=True, slots=True) +class PackagedPolicyEvidence: + """Validated provenance for one exact packaged route table.""" + + route_table_sha256: str + runtime_implementation_sha256: str + model_ids: tuple[str, ...] + dataset_fingerprints: tuple[str, ...] + run_fingerprints: tuple[str, ...] + task_count: int + correct_count: int + total_fireworks_tokens: int + accuracy_semantics: str + + @classmethod + def from_json(cls, path: Path, route_table: RouteTable) -> PackagedPolicyEvidence: + """Load and fully validate an evidence sidecar against runtime behavior.""" + + payload = _load_payload(path) + _require_exact_keys(payload, _TOP_LEVEL_KEYS, "packaged policy evidence") + if _require_integer(payload["schema_version"], "schema_version", 1) != 1: + raise ValueError("packaged policy evidence schema_version must be 1") + + route_table_sha256 = _require_sha256(payload["route_table_sha256"], "route_table_sha256") + if route_table_sha256 != route_table.canonical_sha256(): + raise ValueError("packaged policy evidence does not match the route table SHA-256") + + runtime_implementation_sha256 = _require_sha256( + payload["runtime_implementation_sha256"], "runtime_implementation_sha256" + ) + if runtime_implementation_sha256 != source_tree_fingerprint(): + raise ValueError("packaged policy evidence does not match the runtime implementation") + if payload["prompt_policy"] != PROMPT_POLICY: + raise ValueError("packaged policy evidence uses a different prompt policy") + if payload["normalization_policy"] != NORMALIZATION_POLICY: + raise ValueError("packaged policy evidence uses a different normalization policy") + + model_ids = _require_string_list(payload["model_ids"], "model_ids", allow_empty=True) + if frozenset(model_ids) != route_table.exact_fireworks_model_ids(): + raise ValueError( + "packaged policy evidence model_ids must exactly match fw_model routes" + ) + unsupported_routes = { + name + for name in route_table.configured_route_names() + if name != "deterministic" and not name.startswith("fw_model:") + } + if unsupported_routes: + raise ValueError( + "packaged policy contains a route without exact-model evidence: " + + ", ".join(sorted(unsupported_routes)) + ) + if payload["model_identity_verified"] is not True: + raise ValueError("packaged policy requires verified requested/served model identity") + + dataset_fingerprints = _require_sha256_list( + payload["dataset_fingerprints"], "dataset_fingerprints" + ) + run_fingerprints = _require_sha256_list(payload["run_fingerprints"], "run_fingerprints") + + accuracy = _require_object(payload["accuracy_evidence"], "accuracy_evidence") + _require_exact_keys( + accuracy, + {"semantics", "task_count", "correct_count"}, + "accuracy_evidence", + ) + semantics = accuracy["semantics"] + if not isinstance(semantics, str) or semantics not in _ACCURACY_SEMANTICS: + raise ValueError("accuracy_evidence.semantics is not recognized") + task_count = _require_integer(accuracy["task_count"], "accuracy_evidence.task_count", 1) + correct_count = _require_integer( + accuracy["correct_count"], "accuracy_evidence.correct_count", 0 + ) + if correct_count > task_count: + raise ValueError("accuracy_evidence.correct_count cannot exceed task_count") + + tokens = _require_object(payload["token_evidence"], "token_evidence") + _require_exact_keys( + tokens, + { + "complete", + "failed_attempts_included", + "task_count", + "total_fireworks_tokens", + }, + "token_evidence", + ) + if tokens["complete"] is not True: + raise ValueError("packaged policy requires complete token evidence") + if tokens["failed_attempts_included"] is not True: + raise ValueError("packaged policy token evidence must include failed attempts") + token_task_count = _require_integer(tokens["task_count"], "token_evidence.task_count", 1) + if token_task_count != task_count: + raise ValueError("accuracy and token evidence task counts must match") + total_fireworks_tokens = _require_integer( + tokens["total_fireworks_tokens"], + "token_evidence.total_fireworks_tokens", + 0, + ) + + return cls( + route_table_sha256=route_table_sha256, + runtime_implementation_sha256=runtime_implementation_sha256, + model_ids=model_ids, + dataset_fingerprints=dataset_fingerprints, + run_fingerprints=run_fingerprints, + task_count=task_count, + correct_count=correct_count, + total_fireworks_tokens=total_fireworks_tokens, + accuracy_semantics=semantics, + ) + + +def _load_payload(path: Path) -> dict[str, Any]: + try: + size = path.stat().st_size + if size > _MAX_EVIDENCE_BYTES: + raise ValueError("packaged policy evidence is unexpectedly large") + payload = json.loads(path.read_text(encoding="utf-8")) + except OSError: + raise + except json.JSONDecodeError as exc: + raise ValueError("packaged policy evidence is not valid JSON") from exc + return _require_object(payload, "packaged policy evidence") + + +def _require_object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise ValueError(f"{label} must be a JSON object") + return value + + +def _require_exact_keys(value: dict[str, Any], expected: set[str], label: str) -> None: + actual = set(value) + if actual == expected: + return + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + details: list[str] = [] + if missing: + details.append("missing " + ", ".join(missing)) + if unexpected: + details.append("unexpected " + ", ".join(unexpected)) + raise ValueError(f"{label} has invalid fields ({'; '.join(details)})") + + +def _require_sha256(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise ValueError(f"{label} must be a lowercase SHA-256 digest") + return value + + +def _require_sha256_list(value: object, label: str) -> tuple[str, ...]: + values = _require_string_list(value, label, allow_empty=False) + for item in values: + _require_sha256(item, f"{label} entry") + return values + + +def _require_string_list( + value: object, + label: str, + *, + allow_empty: bool, +) -> tuple[str, ...]: + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ValueError(f"{label} must be an array of strings") + values = tuple(value) + if not allow_empty and not values: + raise ValueError(f"{label} cannot be empty") + if any(not item or item != item.strip() for item in values): + raise ValueError(f"{label} entries must be nonblank and trimmed") + if len(set(values)) != len(values): + raise ValueError(f"{label} cannot contain duplicates") + return values + + +def _require_integer(value: object, label: str, minimum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{label} must be an integer of at least {minimum}") + return value diff --git a/src/fallthrough/policy/route_table.py b/src/fallthrough/policy/route_table.py index 0d5a120..6546d5b 100644 --- a/src/fallthrough/policy/route_table.py +++ b/src/fallthrough/policy/route_table.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from collections.abc import Mapping, Sequence from dataclasses import dataclass @@ -30,19 +31,67 @@ def __post_init__(self) -> None: def for_kind(self, kind: TaskKind) -> tuple[str, ...]: return self.routes.get(kind, self.unknown_routes) + def configured_route_names(self) -> frozenset[str]: + """Return every route named by this table, including the fallback.""" + + return frozenset( + name + for route_names in (*self.routes.values(), self.unknown_routes) + for name in route_names + ) + + def exact_fireworks_model_ids(self) -> frozenset[str]: + """Return exact model IDs used by evidence-bindable Fireworks routes.""" + + prefix = "fw_model:" + model_ids: set[str] = set() + for route_name in self.configured_route_names(): + if not route_name.startswith(prefix): + continue + model_id = route_name[len(prefix) :] + if not model_id or model_id != model_id.strip(): + raise ValueError(f"invalid exact Fireworks route name: {route_name!r}") + model_ids.add(model_id) + return frozenset(model_ids) + + def canonical_payload(self) -> dict[str, object]: + """Return the stable policy payload used for evidence hashing.""" + + return { + "routes": { + kind.value: list(names) + for kind, names in sorted(self.routes.items(), key=lambda item: item[0].value) + }, + "unknown_routes": list(self.unknown_routes), + } + + def canonical_sha256(self) -> str: + """Hash only routing behavior, independent of path or JSON formatting.""" + + encoded = json.dumps( + self.canonical_payload(), + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + @classmethod - def baseline(cls) -> RouteTable: + def baseline(cls, remote_route_name: str = "fw_gemma") -> RouteTable: # This implements handoff Version 1, not a claim of measured optimality. # Deterministic-first kinds cost zero tokens when their solver commits - # and fall through cleanly when it declines. - deterministic_first = ("deterministic", "fw_gemma") + # and fall through cleanly when it declines. Gemma remains preferred + # when available, but the caller can supply another actually allowlisted + # Fireworks route instead of requiring an unavailable model. + remote = _validate_route_names((remote_route_name,), "remote_route_name")[0] + deterministic_first = ("deterministic", remote) return cls( routes={ TaskKind.ARITHMETIC: deterministic_first, TaskKind.STRUCTURED: deterministic_first, TaskKind.LOGIC: deterministic_first, }, - unknown_routes=("fw_gemma",), + unknown_routes=(remote,), ) @classmethod @@ -59,10 +108,18 @@ def from_json(cls, path: Path) -> RouteTable: if not isinstance(raw_names, list): raise ValueError(f"routes.{kind.value} must be an array") routes[kind] = _validate_route_names(raw_names, f"routes.{kind.value}") - raw_unknown = payload.get("unknown_routes", ["fw_gemma"]) + if "unknown_routes" not in payload: + raise ValueError("route table must declare unknown_routes explicitly") + raw_unknown = payload["unknown_routes"] if not isinstance(raw_unknown, list): raise ValueError("unknown_routes must be an array") unknown = _validate_route_names(raw_unknown, "unknown_routes") + configured_names = {name for names in routes.values() for name in names} | set(unknown) + if "fw_gemma" in configured_names: + raise ValueError( + "measured route tables must bind Gemma with fw_model:; " + "fw_gemma is reserved for the unmeasured baseline" + ) return cls(routes=routes, unknown_routes=unknown, source=str(path)) diff --git a/src/fallthrough/routes/deterministic.py b/src/fallthrough/routes/deterministic.py index 791c6c1..efff673 100644 --- a/src/fallthrough/routes/deterministic.py +++ b/src/fallthrough/routes/deterministic.py @@ -17,6 +17,7 @@ from fallthrough.solvers.structured import solve_structured_detailed from fallthrough.tasks.kinds import TaskKind from fallthrough.tasks.models import Task +from fallthrough.tasks.requirements import requests_explanation from fallthrough.validation.arithmetic import ArithmeticValidator _SUPPORTED_KINDS = frozenset({TaskKind.ARITHMETIC, TaskKind.STRUCTURED, TaskKind.LOGIC}) @@ -37,6 +38,8 @@ def supports(self, kind: TaskKind) -> bool: return kind in _SUPPORTED_KINDS def objective_kind(self, task: Task) -> TaskKind | None: + if requests_explanation(task.prompt): + return None if self._arithmetic.supports(task): return TaskKind.ARITHMETIC if solve_structured_detailed(task) is not None: @@ -49,6 +52,8 @@ async def execute(self, task: Task, kind: TaskKind) -> RouteResult: started = time.perf_counter() if not self.supports(kind): return self._decline(started, "unsupported_task_kind") + if requests_explanation(task.prompt): + return self._decline(started, "requested_explanation_not_supported") if kind is TaskKind.ARITHMETIC: result = self._execute_arithmetic(task, started) diff --git a/src/fallthrough/routes/fireworks_gemma.py b/src/fallthrough/routes/fireworks_gemma.py index f69ac7d..c29c008 100644 --- a/src/fallthrough/routes/fireworks_gemma.py +++ b/src/fallthrough/routes/fireworks_gemma.py @@ -27,6 +27,8 @@ def __init__( temperature: float = 0.0, supported_kinds: Collection[TaskKind] | None = None, format_retries: int = 1, + reasoning_effort: str | int | bool | None = None, + prompt_suffix: str | None = None, ) -> None: super().__init__( client, @@ -36,6 +38,8 @@ def __init__( supported_kinds=supported_kinds, route_name=self.name, format_retries=format_retries, + reasoning_effort=reasoning_effort, + prompt_suffix=prompt_suffix, ) diff --git a/src/fallthrough/routes/fireworks_model.py b/src/fallthrough/routes/fireworks_model.py index 00bb088..9160934 100644 --- a/src/fallthrough/routes/fireworks_model.py +++ b/src/fallthrough/routes/fireworks_model.py @@ -57,6 +57,8 @@ def __init__( supported_kinds: Collection[TaskKind] | None = None, route_name: str | None = None, format_retries: int = 1, + reasoning_effort: str | int | bool | None = None, + prompt_suffix: str | None = None, ) -> None: # Reject a bad configured model before processing any benchmark task. client.validate_model(model) @@ -67,13 +69,23 @@ def __init__( self._supported_kinds = None if supported_kinds is None else frozenset(supported_kinds) self.name = route_name or type(self).name self._format_retries = max(0, format_retries) + self._reasoning_effort = ( + "none" if reasoning_effort is None and "kimi" in model.casefold() else reasoning_effort + ) + self._prompt_suffix = prompt_suffix + self._prompt_suffix_policy = ( + _inference.KIMI_PROMPT_POLICY + if prompt_suffix is None and "kimi" in model.casefold() + else "configured-static" + ) def supports(self, kind: TaskKind) -> bool: return self._supported_kinds is None or kind in self._supported_kinds async def execute(self, task: Task, kind: TaskKind) -> RouteResult: started = time.perf_counter() - prompt = build_task_prompt(task, kind) + prompt_suffix = self._prompt_suffix_for(task, kind) + prompt = build_task_prompt(task, kind) + prompt_suffix attempts_allowed = 1 + self._format_retries total_prompt_tokens = 0 total_completion_tokens = 0 @@ -98,12 +110,13 @@ def failure(metadata: dict[str, Any], *, latency_ms: float | None = None) -> Rou for attempt in range(attempts_allowed): retries = attempt try: - response = await self._client.complete( - self.model, - prompt, - max_tokens=self._max_tokens_for(kind), - temperature=self._temperature, - ) + completion_options: dict[str, Any] = { + "max_tokens": self._max_tokens_for(kind), + "temperature": self._temperature, + } + if self._reasoning_effort is not None: + completion_options["reasoning_effort"] = self._reasoning_effort + response = await self._client.complete(self.model, prompt, **completion_options) except FireworksError as exc: total_prompt_tokens += exc.prompt_tokens total_completion_tokens += exc.completion_tokens @@ -157,7 +170,7 @@ def failure(metadata: dict[str, Any], *, latency_ms: float | None = None) -> Rou empty_code = "```" in answer rejection = "code fence contains no code" if empty_code else "empty completion" if attempt + 1 < attempts_allowed: - prompt = corrective_prompt(task, kind, rejection) + prompt = corrective_prompt(task, kind, rejection) + prompt_suffix continue return failure( { @@ -190,7 +203,7 @@ def failure(metadata: dict[str, Any], *, latency_ms: float | None = None) -> Rou validation_metadata = asdict(validation) if validation is not None else None if validation is not None and not validation.valid: if attempt + 1 < attempts_allowed: - prompt = corrective_prompt(task, kind, validation.reason) + prompt = corrective_prompt(task, kind, validation.reason) + prompt_suffix continue return failure( { @@ -214,13 +227,25 @@ def failure(metadata: dict[str, Any], *, latency_ms: float | None = None) -> Rou latency_ms=response.latency_ms if attempt == 0 else _elapsed_ms(started), metadata={ "model": response.model, + "requested_model": self.model, + "model_identity_match": response.model == self.model, "finish_reason": response.finish_reason, + "reasoning_effort": self._reasoning_effort, + "prompt_suffix": prompt_suffix, + "prompt_suffix_policy": self._prompt_suffix_policy, "validator": validation_metadata, "retries": retries, }, ) raise AssertionError("unreachable: attempt loop always returns") # pragma: no cover + def _prompt_suffix_for(self, task: Task, kind: TaskKind) -> str: + if self._prompt_suffix is not None: + return self._prompt_suffix + if "kimi" in self.model.casefold(): + return _inference.kimi_prompt_suffix(task, kind) + return "" + def _max_tokens_for(self, kind: TaskKind) -> int: configured = self._max_tokens if isinstance(configured, int): diff --git a/src/fallthrough/routes/local_model.py b/src/fallthrough/routes/local_model.py index fbcd185..2c5d341 100644 --- a/src/fallthrough/routes/local_model.py +++ b/src/fallthrough/routes/local_model.py @@ -8,7 +8,7 @@ from fallthrough.inference import ( DEFAULT_MAX_TOKENS_BY_KIND, build_task_prompt, - normalize_code_answer, + canonicalize_answer, ) from fallthrough.local import LocalBackend, stable_model_label from fallthrough.tasks.kinds import TaskKind @@ -59,7 +59,7 @@ async def execute(self, task: Task, kind: TaskKind) -> RouteResult: }, ) - candidate = normalize_code_answer(task, kind, generation.text.strip()) + candidate = canonicalize_answer(task, kind, generation.text.strip()) failure: str | None try: validation = validate_candidate(task, kind, candidate) @@ -96,10 +96,10 @@ async def execute(self, task: Task, kind: TaskKind) -> RouteResult: "prompt_tokens": generation.prompt_token_count, "tokens_generated": generation.token_count, "finish_reason": generation.finish_reason, - "peak_vram_bytes": generation.peak_vram_bytes, - "vram_measurement": generation.metadata.get( - "vram_measurement", - getattr(self.backend, "vram_measurement", None), + "peak_memory_bytes": generation.peak_memory_bytes, + "memory_measurement": generation.metadata.get( + "memory_measurement", + getattr(self.backend, "memory_measurement", None), ), "cumulative_logprob": cumulative_logprob, "mean_logprob": mean_logprob, diff --git a/src/fallthrough/runtime/adapter.py b/src/fallthrough/runtime/adapter.py index 93d9417..03090f5 100644 --- a/src/fallthrough/runtime/adapter.py +++ b/src/fallthrough/runtime/adapter.py @@ -1,8 +1,10 @@ -"""Narrow boundary around the evaluator's JSON schema. +"""Narrow boundary around the official evaluator JSON schema. -The official ACT II schema was not present in the handoff or repository. This -adapter therefore supports an explicit field mapping and a conservative set of -common JSON shapes. No schema assumptions leak into the runtime. +ACT II Track 1 uses a top-level list of ``task_id``/``prompt`` objects and +expects a top-level list of ``task_id``/``answer`` objects. The adapter keeps +the earlier explicit mappings and conservative compatibility shapes at this +boundary so development tools can vary without leaking schema logic into the +runtime. """ from __future__ import annotations @@ -28,7 +30,7 @@ class AdapterError(ValueError): class EvaluationAdapter: """Load evaluator JSON into internal models and atomically write answers.""" - _ID_CANDIDATES = ("id", "task_id") + _ID_CANDIDATES = ("task_id", "id") _PROMPT_CANDIDATES = ("prompt", "question", "input") def __init__( @@ -77,11 +79,12 @@ def __init__( def _read_text(self) -> str: if self.input_path is not None: try: - return self.input_path.read_text(encoding="utf-8") + return self.input_path.read_text(encoding="utf-8-sig") except OSError as exc: raise AdapterError(f"cannot read evaluator input: {exc}") from exc stream = self.input_stream or sys.stdin - return stream.read() + text = stream.read() + return text[1:] if text.startswith("\ufeff") else text def load_tasks(self) -> list[Task]: try: diff --git a/src/fallthrough/runtime/engine.py b/src/fallthrough/runtime/engine.py index baec4e2..615353c 100644 --- a/src/fallthrough/runtime/engine.py +++ b/src/fallthrough/runtime/engine.py @@ -28,7 +28,8 @@ def __init__( router: BaselineRouter, trace_sink: TraceSink | None = None, global_timeout_seconds: float = 270.0, - per_route_timeout_seconds: float = 30.0, + per_task_timeout_seconds: float = 29.0, + per_route_timeout_seconds: float = 25.0, max_concurrency: int = 8, failure_answer: str = "", ) -> None: @@ -38,10 +39,15 @@ def __init__( raise ValueError("max_concurrency must be positive") if not math.isfinite(per_route_timeout_seconds) or per_route_timeout_seconds <= 0: raise ValueError("per_route_timeout_seconds must be positive") + if not math.isfinite(per_task_timeout_seconds) or not 0 < per_task_timeout_seconds < 30: + raise ValueError("per_task_timeout_seconds must be positive and below 30") + if per_route_timeout_seconds > per_task_timeout_seconds: + raise ValueError("per_route_timeout_seconds cannot exceed per_task_timeout_seconds") self._routes = routes self._router = router self._trace = SafeTraceSink(trace_sink or NullTraceSink()) self._global_timeout_seconds = global_timeout_seconds + self._per_task_timeout_seconds = per_task_timeout_seconds self._per_route_timeout_seconds = per_route_timeout_seconds self._max_concurrency = max_concurrency self._failure_answer = failure_answer @@ -64,18 +70,29 @@ async def solve_batch( results: list[TaskResult | None] = [None] * len(tasks) async def worker(index: int, task: Task) -> tuple[int, TaskResult]: - async with semaphore: - try: - return index, await self._solve_task( - task, - active_deadline, - reserve_seconds=reserve_seconds, + try: + async with semaphore: + # A queued task has not started processing yet. Give every + # admitted task its own sub-30-second execution budget while + # the batch-wide deadline still bounds queueing and total time. + task_deadline = Deadline(self._per_task_timeout_seconds) + return index, await asyncio.wait_for( + self._solve_task( + task, + active_deadline, + task_deadline=task_deadline, + reserve_seconds=reserve_seconds, + ), + timeout=task_deadline.remaining_seconds, ) - except asyncio.CancelledError: - raise - except Exception as exc: # absolute task-level isolation - self._emit_failure(task, "unexpected_task_error", exc) - return index, self._fallback(task) + except asyncio.TimeoutError: + self._emit_task_deadline(task) + return index, self._fallback(task) + except asyncio.CancelledError: + raise + except Exception as exc: # absolute task-level isolation + self._emit_failure(task, "unexpected_task_error", exc) + return index, self._fallback(task) pending_tasks = { asyncio.create_task(worker(index, task)): (index, task) @@ -124,6 +141,7 @@ async def _solve_task( task: Task, deadline: Deadline, *, + task_deadline: Deadline, reserve_seconds: float, ) -> TaskResult: started = time.perf_counter() @@ -144,9 +162,12 @@ async def _solve_task( selected: RouteResult | None = None for route_name in route_names: - route_budget = deadline.clamp( - self._per_route_timeout_seconds, - reserve_seconds=reserve_seconds, + route_budget = min( + task_deadline.remaining_seconds, + deadline.clamp( + self._per_route_timeout_seconds, + reserve_seconds=reserve_seconds, + ), ) if route_budget <= 0: attempts.append({"route_name": route_name, "success": False, "reason": "deadline"}) @@ -256,6 +277,20 @@ def _emit_failure(self, task: Task, failure: str, exc: Exception) -> None: } ) + def _emit_task_deadline(self, task: Task) -> None: + self._trace.emit( + { + "event": "task_completed", + "task_id": task.id, + "success": False, + "failure": "task_deadline_exceeded", + "selected_route": None, + "fireworks_prompt_tokens": 0, + "fireworks_completion_tokens": 0, + "fireworks_usage_known": False, + } + ) + def _failure_reason(result: RouteResult) -> str: reason = result.metadata.get("reason") diff --git a/src/fallthrough/tasks/classifier.py b/src/fallthrough/tasks/classifier.py index 88258ae..6b002db 100644 --- a/src/fallthrough/tasks/classifier.py +++ b/src/fallthrough/tasks/classifier.py @@ -28,6 +28,11 @@ r"(?:an?\s+|the\s+)?.+\?\s*$", re.IGNORECASE, ) +_FACTUAL_COMPARISON_RE = re.compile( + r"^\s*(?:(?:what\s+(?:is|are)|explain|describe)\s+)?" + r"(?:the\s+)?differences?\s+between\b", + re.IGNORECASE, +) _TASK_TERM = ( r"(?:sentiment\s+analysis|named\s+entit(?:y\s+recognition|ies)|ner|" r"stack\s+trace|traceback)" @@ -48,6 +53,21 @@ r"following\s+(?:items|reviews|texts|statements|numbers))\b", re.IGNORECASE, ) +_DISTINCT_ASSIGNMENT_RE = re.compile( + r"\beach\b[^.\n]{0,100}\b(?:a\s+)?different\b|" + r"\b(?:one-to-one|unique)\b[^.\n]{0,60}\b(?:assignment|match|pairing)\b", + re.IGNORECASE, +) +_ASSIGNMENT_QUERY_RE = re.compile( + r"\b(?:who|which\s+(?:person|one|item))\b[^?\n]{0,80}" + r"\b(?:owns?|has|gets?|is\s+assigned|belongs?\s+to)\b", + re.IGNORECASE, +) +_ASSIGNMENT_CLUE_RE = re.compile( + r"\b(?:does\s+not|doesn't|is\s+not|isn't|owns?|has|gets?|" + r"is\s+assigned|belongs?\s+to)\b", + re.IGNORECASE, +) def _logged(task: Task, features: TaskFeatures, kind: TaskKind, reason: str) -> TaskKind: @@ -93,6 +113,13 @@ def classify_task(task: Task, features: TaskFeatures) -> TaskKind: if features.has_traceback or (features.asks_code_debug and features.has_code): return _logged(task, features, TaskKind.CODE_DEBUG, "traceback or explicit code repair") + if ( + _DISTINCT_ASSIGNMENT_RE.search(prompt) + and _ASSIGNMENT_QUERY_RE.search(prompt) + and len(_ASSIGNMENT_CLUE_RE.findall(prompt)) >= 2 + ): + return _logged(task, features, TaskKind.LOGIC, "distinct assignment constraints") + if features.logic_score >= 0.50: return _logged(task, features, TaskKind.LOGIC, "multiple logic/constraint signals") @@ -119,9 +146,16 @@ def classify_task(task: Task, features: TaskFeatures) -> TaskKind: re.match(r"^\s*(?:what|which|how|is|are|was|were)\b", prompt, re.IGNORECASE) ) has_definition_form = bool(_DEFINITION_QUESTION_RE.search(prompt)) + has_comparison_form = bool(_FACTUAL_COMPARISON_RE.search(prompt)) if features.is_question and ( - has_unambiguously_factual_form or has_factual_subject or has_definition_form + has_unambiguously_factual_form + or has_factual_subject + or has_definition_form + or has_comparison_form ): return _logged(task, features, TaskKind.FACTUAL_QA, "fact-seeking question form") + if has_comparison_form: + return _logged(task, features, TaskKind.FACTUAL_QA, "factual comparison request") + return _logged(task, features, TaskKind.UNKNOWN, "no sufficiently specific rule") diff --git a/src/fallthrough/tasks/features.py b/src/fallthrough/tasks/features.py index 6102855..8b9c96d 100644 --- a/src/fallthrough/tasks/features.py +++ b/src/fallthrough/tasks/features.py @@ -6,7 +6,11 @@ import re from dataclasses import dataclass -from fallthrough.tasks.labels import sentiment_labels_from_metadata, sentiment_labels_from_prompt +from fallthrough.tasks.labels import ( + is_sentiment_meta_question, + sentiment_labels_from_metadata, + sentiment_labels_from_prompt, +) from fallthrough.tasks.models import Task _TOKEN_RE = re.compile(r"(? TaskFeatures: has_function_syntax = bool(_FUNCTION_SYNTAX_RE.search(prompt)) has_class_syntax = bool(_CLASS_SYNTAX_RE.search(prompt)) has_traceback = bool(_TRACEBACK_RE.search(prompt)) - has_code = has_code_fence or has_function_syntax or has_class_syntax or has_traceback + has_code = ( + has_code_fence + or has_function_syntax + or has_class_syntax + or has_traceback + or bool(_INLINE_CODE_RE.search(prompt)) + ) arithmetic_words = len(_ARITHMETIC_WORD_RE.findall(prompt)) has_arithmetic_request = bool(_ARITHMETIC_REQUEST_RE.search(prompt)) @@ -433,7 +451,7 @@ def extract_features(task_or_prompt: Task | str) -> TaskFeatures: ) prompt_labels = sentiment_labels_from_prompt(prompt) asks_sentiment = ( - bool(_SENTIMENT_REQUEST_RE.search(prompt)) + (not is_sentiment_meta_question(prompt) and bool(_SENTIMENT_REQUEST_RE.search(prompt))) or bool( metadata_labels and re.search( diff --git a/src/fallthrough/tasks/labels.py b/src/fallthrough/tasks/labels.py index e53ada8..f5ff6c3 100644 --- a/src/fallthrough/tasks/labels.py +++ b/src/fallthrough/tasks/labels.py @@ -7,6 +7,8 @@ from collections.abc import Mapping from typing import Any +from fallthrough.tasks.requirements import instruction_regions + SENTIMENT_LABEL_KEYS = ( "allowed_labels", "sentiment_labels", @@ -14,15 +16,12 @@ "output_labels", ) -_CLASSIFY_AS_LABELS_RE = re.compile( +_CLASSIFY_LABELS_RE = re.compile( r"\bclassify" r"(?P\s+(?:(?:the|this)\s+)?(?:following\s+)?" r"(?:sentiment|review|text|statement|opinion|input))?" - r"\s+as\s+" - r"(?P[A-Za-z0-9_-]{1,64}(?:\s+[A-Za-z0-9_-]{1,64}){0,2}" - r"(?:\s+or\s+|\s*/\s*)" - r"[A-Za-z0-9_-]{1,64}(?:\s+[A-Za-z0-9_-]{1,64}){0,2})" - r"(?=\s*[:.;\n]|$)", + r"\s*(?:as|:)\s*" + r"(?P[^\n:.;]{1,256})", re.IGNORECASE, ) _POLARITY_LABEL_HINTS = frozenset( @@ -43,12 +42,38 @@ "bullish", } ) -_SOURCE_DELIMITER_RE = re.compile( - r"\b(?:review|text|input|sentence|passage|article|document)\s*:", +_POLARITY_CHOICE = ( + r"(?:very\s+positive|very\s+negative|positive|negative|neutral|mixed|favorable|" + r"favourable|unfavorable|unfavourable|good|bad|bullish|bearish)" +) +_CHOICE_LABELS_RE = re.compile( + rf"(?:\A|[\r\n])\s*(?:please\s+)?(?:choose\s+between|select)\s+" + rf"(?P{_POLARITY_CHOICE})\s+(?:and|or|/)\s+" + rf"(?P{_POLARITY_CHOICE})\b|" + rf"(?:\A|[\r\n])\s*is\s+(?:this|the)\s+(?:review|text|sentiment|opinion)\s+" + rf"(?P{_POLARITY_CHOICE})\s+or\s+" + rf"(?P{_POLARITY_CHOICE})\b|" + rf"(?:\A|[\r\n])\s*how\s+should\s+(?:this|the)\s+" + rf"(?:review|text|sentiment|opinion)\s+be\s+classified\s*:\s*" + rf"(?P{_POLARITY_CHOICE})\s+or\s+" + rf"(?P{_POLARITY_CHOICE})\b", + re.IGNORECASE, +) +_META_SENTIMENT_QUERY_RE = re.compile( + r"^\s*(?:what\s+does\s+it\s+mean\s+to|explain\s+(?:how|why)\b|" + r"when\s+should\b|how\s+(?:do|should|can)\s+(?:i|you|users?)\b)", + re.IGNORECASE, +) +_LATER_SENTIMENT_TASK_RE = re.compile( + r"[\r\n]+\s*(?:please\s+)?(?:classify|select|choose\s+between|" + r"is\s+(?:this|the)\s+(?:review|text|sentiment|opinion))\b", re.IGNORECASE, ) -_LABEL_DECLARATION_RE = re.compile( - r"\b(?:labels?|one\s+of|choose\s+from)\s*$", +_LABEL_REQUIREMENT_SUFFIX_RE = re.compile( + r"\s+and\s+(?=(?:briefly\s+)?(?:give|provide|include|add|explain|justify|" + r"state|support)\b)|" + r"\s+with\s+(?=(?:(?:a|an|the)\s+)?(?:reason|explanation|justification|" + r"rationale|supporting\s+evidence)\b)", re.IGNORECASE, ) @@ -96,14 +121,40 @@ def sentiment_labels_from_prompt(prompt: str) -> tuple[str, ...]: if not isinstance(prompt, str) or len(prompt) > 1_000_000: return () instruction = sentiment_instruction_header(prompt) - match = _CLASSIFY_AS_LABELS_RE.search(instruction) + if is_sentiment_meta_question(instruction): + return () + choice_match = _CHOICE_LABELS_RE.search(instruction) + if choice_match is not None: + first = ( + choice_match.group("first") + or choice_match.group("question_first") + or choice_match.group("passive_first") + ) + second = ( + choice_match.group("second") + or choice_match.group("question_second") + or choice_match.group("passive_second") + ) + return first, second + match = _CLASSIFY_LABELS_RE.search(instruction) if match is None: return () + clause = _LABEL_REQUIREMENT_SUFFIX_RE.split(match.group("labels"), maxsplit=1)[0] + clause = re.sub(r"^\s*(?:one\s+of|either)\s+", "", clause, flags=re.IGNORECASE) + clause = re.sub(r"\s+(?:labels?\s+)?only\s*$", "", clause, flags=re.IGNORECASE) labels = tuple( part.strip() - for part in re.split(r"\s+or\s+|\s*/\s*", match.group("labels"), flags=re.IGNORECASE) + for part in re.split(r"\s*(?:,|/|\||\bor\b)\s*", clause, flags=re.IGNORECASE) + if part.strip() ) - if not 2 <= len(labels) <= 4 or len({label.casefold() for label in labels}) != len(labels): + if ( + not 2 <= len(labels) <= 4 + or len({label.casefold() for label in labels}) != len(labels) + or any( + re.fullmatch(r"[A-Za-z0-9_-]+(?: [A-Za-z0-9_-]+){0,2}", label) is None + for label in labels + ) + ): return () target = (match.group("target") or "").casefold() has_sentiment_target = "sentiment" in target @@ -114,8 +165,18 @@ def sentiment_labels_from_prompt(prompt: str) -> tuple[str, ...]: return labels +def is_sentiment_meta_question(prompt: str) -> bool: + """Identify explanation/advice prompts about sentiment classification.""" + + return bool( + isinstance(prompt, str) + and _META_SENTIMENT_QUERY_RE.search(prompt) + and _LATER_SENTIMENT_TASK_RE.search(prompt) is None + ) + + def sentiment_instruction_header(prompt: str) -> str: - """Return the task instruction without labelled or delimited source text.""" + """Return bounded leading and source-following sentiment instructions.""" if not isinstance(prompt, str): return "" @@ -126,15 +187,4 @@ def sentiment_instruction_header(prompt: str) -> str: count=1, flags=re.IGNORECASE, ) - cutoff = len(prompt) - source = _SOURCE_DELIMITER_RE.search(prompt) - if source is not None: - cutoff = min(cutoff, source.start()) - newline = re.search(r"\r?\n", prompt) - if newline is not None: - cutoff = min(cutoff, newline.start()) - instruction = prompt[:cutoff] - colon = instruction.find(":") - if colon >= 0 and _LABEL_DECLARATION_RE.search(instruction[:colon]) is None: - instruction = instruction[:colon] - return instruction.strip() + return "\n".join(instruction_regions(prompt)) diff --git a/src/fallthrough/tasks/requirements.py b/src/fallthrough/tasks/requirements.py new file mode 100644 index 0000000..b03af0f --- /dev/null +++ b/src/fallthrough/tasks/requirements.py @@ -0,0 +1,160 @@ +"""Small, bounded detectors for explicit answer requirements.""" + +from __future__ import annotations + +import re + +_EXPLANATION_TERM = ( + r"(?:explain(?:ing|ed)?|explanation|reason(?:ing)?|justify|justification|" + r"diagnos(?:e|is))" +) +_NEGATED_EXPLANATION_RE = re.compile( + rf"\b(?:(?:do\s+not|don't|never|omit|avoid)\s+" + rf"(?:(?:include|provide|give|add|show)\s+)?" + rf"(?:(?:an?|any|the|your)\s+)?{_EXPLANATION_TERM}|" + rf"without\s+(?:(?:an?|any|the|your)\s+)?{_EXPLANATION_TERM}|" + rf"no\s+{_EXPLANATION_TERM}|" + rf"(?:instead\s+of|rather\s+than|not)\s+" + rf"(?:(?:an?|any|the|your)\s+)?{_EXPLANATION_TERM})\b", + re.IGNORECASE, +) +_EXPLANATION_REQUEST_RE = re.compile( + rf"\b{_EXPLANATION_TERM}\b|" + r"\bshow\b[^.\n]{0,24}\b(?:work|steps?)\b|" + r"\b(?:what|why)\b[^?\n]{0,60}\b(?:wrong|bug|fails?|happens?)\b", + re.IGNORECASE, +) +_SOURCE_DELIMITER_RE = re.compile( + r"\b(?:article|code|document|input|passage|program|review|sentence|snippet|source|text)\s*:", + re.IGNORECASE, +) +_CODE_START_RE = re.compile( + r"(?:^|\n)\s*(?:```|(?:async\s+)?def\s+|class\s+|function\s+|" + r"(?:const|let|var)\s+|for\s*\(|#include\b)", + re.IGNORECASE, +) +_TRAILING_INSTRUCTION_RE = re.compile( + r"(?:\A|[\r\n]+|[.!?;]\s+)" + r"(?P\s*" + r"(?:(?:also|then|next|finally)\s+|" + r"after\s+(?:the\s+)?(?:code|answer|result)\s*,?\s+|" + r"you\s+(?:should|must|need\s+to)\s+)?" + r"(?:please\s+)?(?:briefly\s+)?" + r"(?:answer|respond|reply|write|return|output|provide|give|include|format|" + r"wrap|enclose|put|generate|create|explain|justify|diagnose|show|summarize|" + r"classify|extract|identify|name|fix|debug|list|state|support|use|what|why)\b" + r"(?=\s|[.:;,!?-]|$)" + r"[^\r\n]{0,500}?)\s*\Z", + re.IGNORECASE, +) +_NEGATED_FENCE_REQUEST_RE = re.compile( + r"\b(?:do\s+not|don't|without|no|never|avoid|omit)\b[^.;\n]{0,60}" + r"(?:\b(?:a\s+)?(?:markdown\s+)?code\s+(?:block|fence)s?\b|" + r"\bfenced\s+(?:python\s+)?code(?:\s+block)?\b|" + r"\bmarkdown(?:\s+(?:fence|formatting|markup))?\b|" + r"\b(?:triple\s+)?backticks?\b)|" + r"\b(?:instead\s+of|rather\s+than|not)\s+(?:a\s+)?" + r"(?:(?:markdown\s+)?code\s+(?:block|fence)s?|markdown\s+fence|" + r"fenced\s+(?:python\s+)?code(?:\s+block)?|(?:triple\s+)?backticks?)\b", + re.IGNORECASE, +) +_FENCE_TERM = ( + r"(?:(?:markdown\s+)?code\s+(?:block|fence)s?|markdown\s+fence|" + r"fenced\s+(?:python\s+)?code(?:\s+block)?|triple\s+backticks?)" +) +_POSITIVE_FENCE_REQUEST_RE = re.compile( + rf"\b(?:write|return|output|provide|format|wrap|enclose|put|generate|create|" + rf"respond|reply|give|answer)\b[^.\n]{{0,120}}" + rf"\b(?:in|inside|within|as|using|with)\s+(?:a\s+)?{_FENCE_TERM}\b|" + rf"\buse\s+(?:a\s+)?{_FENCE_TERM}\b|" + rf"(?:^|[.;]\s*)(?:return|output|provide|include|wrap|enclose)\s+" + rf"(?:only\s+)?(?:the\s+|a\s+)?{_FENCE_TERM}\b", + re.IGNORECASE | re.MULTILINE, +) +_NON_ENGLISH_OUTPUT_RE = re.compile( + r"\b(?:answer|respond|reply|write|return|output|provide|give)\b" + r"[^.\n]{0,50}\bin\s+(?:the\s+)?" + r"(?:arabic|chinese|dutch|french|german|greek|hindi|italian|japanese|korean|" + r"polish|portuguese|russian|spanish|swedish|turkish)\b|" + r"\btranslate\b[^.\n]{0,80}\binto\s+" + r"(?:arabic|chinese|dutch|french|german|greek|hindi|italian|japanese|korean|" + r"polish|portuguese|russian|spanish|swedish|turkish)\b", + re.IGNORECASE, +) + + +def instruction_header(prompt: str) -> str: + """Return the bounded instruction portion before supplied source material.""" + + if not isinstance(prompt, str): + return "" + cutoff = min(len(prompt), 4096) + source = _SOURCE_DELIMITER_RE.search(prompt) + if source is not None: + cutoff = min(cutoff, source.start()) + code_start = _CODE_START_RE.search(prompt) + if code_start is not None: + cutoff = min(cutoff, code_start.start()) + blank_line = re.search(r"\r?\n\s*\r?\n", prompt) + if blank_line is not None: + cutoff = min(cutoff, blank_line.start()) + return prompt[:cutoff].strip() + + +def instruction_regions(prompt: str) -> tuple[str, ...]: + """Return bounded regions likely to contain evaluator instructions. + + The leading region excludes supplied source material. A final imperative + clause is also retained because evaluator prompts commonly put output + requirements after ``Input:``, ``Passage:``, or a code sample. + """ + + if not isinstance(prompt, str): + return () + bounded_prompt = prompt[:20_000] + regions: list[str] = [] + header = instruction_header(bounded_prompt) + if header: + regions.append(header) + trailing = _TRAILING_INSTRUCTION_RE.search(bounded_prompt) + if trailing is not None: + instruction = trailing.group("instruction").strip() + source = _SOURCE_DELIMITER_RE.search(bounded_prompt) + trailing_is_after_source = source is None or trailing.start("instruction") > source.start() + if instruction and instruction not in regions and trailing_is_after_source: + regions.append(instruction) + return tuple(regions) + + +def requests_explanation(prompt: str) -> bool: + """Whether the evaluator explicitly asks for reasoning or diagnosis.""" + + for instruction in instruction_regions(prompt): + without_negated_requests = _NEGATED_EXPLANATION_RE.sub("", instruction) + if _EXPLANATION_REQUEST_RE.search(without_negated_requests) is not None: + return True + return False + + +def requests_fenced_output(prompt: str) -> bool: + """Whether an instruction explicitly requests a fenced code answer.""" + + instruction = "\n".join(instruction_regions(prompt)) + positive = _POSITIVE_FENCE_REQUEST_RE.search(instruction) + if positive is None: + return False + negative = _NEGATED_FENCE_REQUEST_RE.search(instruction) + return negative is None or positive.start() < negative.start() + + +def forbids_fenced_output(prompt: str) -> bool: + """Whether an instruction explicitly prohibits Markdown code fences.""" + + instruction = "\n".join(instruction_regions(prompt)) + return _NEGATED_FENCE_REQUEST_RE.search(instruction) is not None + + +def requests_non_english_output(prompt: str) -> bool: + """Detect an explicit output-language request that conflicts with Track 1.""" + + return isinstance(prompt, str) and _NON_ENGLISH_OUTPUT_RE.search(prompt[:4096]) is not None diff --git a/src/fallthrough/telemetry/observations.py b/src/fallthrough/telemetry/observations.py index d309cae..2436bc3 100644 --- a/src/fallthrough/telemetry/observations.py +++ b/src/fallthrough/telemetry/observations.py @@ -25,6 +25,20 @@ class RouteObservation: route_features: dict[str, float | int | bool | str] = field(default_factory=dict) answer: str | None = None expected_answer: Any = None + declared_task_kind: str | None = None + classified_task_kind: str | None = None + task_family: str | None = None + dataset_fingerprint: str = "" + dataset_split: str = "unspecified" + run_fingerprint: str = "" + dataset_task_count: int = 0 + implementation_sha256: str = "" + shared_implementation_sha256: str = "" + benchmark_context_fingerprint: str = "" + declared_scorer: str = "auto" + effective_scorer: str = "auto" + scoring_policy: str = "" + run_configuration: dict[str, Any] = field(default_factory=dict) @property def fireworks_tokens(self) -> int: diff --git a/src/fallthrough/validation/__init__.py b/src/fallthrough/validation/__init__.py index d7d7f73..1f487b1 100644 --- a/src/fallthrough/validation/__init__.py +++ b/src/fallthrough/validation/__init__.py @@ -4,7 +4,9 @@ from .base import ValidationResult, Validator from .code import ( CodeValidator, + GenericCodeValidator, PythonCodeValidator, + PythonCodeWithExplanationValidator, extract_python_code, validate_python_syntax, ) @@ -17,19 +19,23 @@ ) from .format import FormatValidator, JSONValidator, JsonValidator, parse_json from .registry import validate_candidate -from .sentiment import SentimentValidator, requested_sentiment_labels +from .sentiment import SentimentValidator, requested_sentiment_labels, sentiment_reason_required +from .summary import SummaryFormatValidator __all__ = [ "ArithmeticValidator", "CodeValidator", "EntityValidator", "FormatValidator", + "GenericCodeValidator", "JSONValidator", "JsonValidator", "NERValidator", "ParsedEntity", "PythonCodeValidator", + "PythonCodeWithExplanationValidator", "SentimentValidator", + "SummaryFormatValidator", "ValidationResult", "Validator", "extract_python_code", @@ -39,6 +45,7 @@ "parse_json", "parse_numeric_answer", "requested_sentiment_labels", + "sentiment_reason_required", "validate_python_syntax", "validate_candidate", ] diff --git a/src/fallthrough/validation/code.py b/src/fallthrough/validation/code.py index 96374a6..e66588b 100644 --- a/src/fallthrough/validation/code.py +++ b/src/fallthrough/validation/code.py @@ -6,6 +6,12 @@ import re from typing import Any +from fallthrough.tasks.requirements import ( + forbids_fenced_output, + instruction_regions, + requests_fenced_output, +) + from .base import ValidationResult, Validator MAX_CODE_LENGTH = 100_000 @@ -14,6 +20,60 @@ r"(?P.*?)\r?\n```\s*\Z", re.DOTALL, ) +_EMBEDDED_FENCE_RE = re.compile( + r"```(?P[A-Za-z0-9_+-]*)[ \t]*\r?\n" + r"(?P.*?)\r?\n```", + re.DOTALL, +) +_GENERIC_CODE_SIGNAL_RE = re.compile( + r"(?:^|\n)\s*(?:#!|#include\b|(?:export\s+)?(?:async\s+)?function\b|" + r"class\b|(?:const|let|var)\b|(?:pub\s+)?fn\b|func\b|package\b|import\b|" + r"using\b|namespace\b|(?:select|insert|update|delete|create|with)\b|" + r"echo\b|printf\b|return\b|(?:if|for|while|switch)\s*\()|" + r"=>|[{};]|" + r"(?:^|\n)\s*]*>|" + r"(?:^|\n)\s*[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*\([^\n]*\)\s*;?\s*(?:\n|$)|" + r"(?:^|\n)\s*[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*[:=]\s*\S", + re.IGNORECASE, +) +_GENERIC_EXPRESSION_RE = re.compile( + r"\A\s*[A-Za-z0-9_$.'\"\[\](),-]+(?:\s*(?:\+|-|\*|/|%|==|!=|<=|>=|<|>|" + r"&&|\|\||\?|:)\s*[A-Za-z0-9_$.'\"\[\](),-]+)+\s*\Z" +) +_SHELL_PROMPT_RE = re.compile(r"\b(?:bash|shell|command(?:-line)?|terminal)\b", re.IGNORECASE) +_SHELL_CODE_RE = re.compile( + r"\A\s*(?:(?:ls|pwd|cd|echo|printf|grep|sed|awk|find|head|tail|sort|uniq|" + r"cut|tr|test|export|read|case)\b|[A-Za-z0-9_./-]+\s+--?[A-Za-z0-9_-]+)" + r"[^\r\n]*\s*\Z", + re.IGNORECASE, +) +_CODE_ONLY_REQUEST_RE = re.compile( + r"\b(?:return|output|provide|give|respond\s+with|reply\s+with)\s+" + r"(?:only\s+)?(?:the\s+)?(?:corrected\s+)?(?:source\s+)?code\s+only\b|" + r"\b(?:return|output|provide|give)\s+only\s+(?:the\s+)?" + r"(?:corrected\s+)?(?:source\s+)?code\b|" + r"\b(?:corrected\s+)?(?:source\s+)?code\s+only\b", + re.IGNORECASE, +) +_CODE_COMMENT_RE = re.compile(r"\A\s*(?://|/\*|