diff --git a/README.md b/README.md index ef6034b..a2d1c95 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,8 @@ PDF compilation requires `tectonic` or `pdflatex`. Configure spending and execut See [Deployment](https://github.com/mikamikasuki/FOREST/blob/main/docs/DEPLOYMENT.md) for Docker, PostgreSQL, background services, backups, and Windows via WSL2 or Docker Desktop. See [Security](https://github.com/mikamikasuki/FOREST/blob/main/SECURITY.md) for execution boundaries. +Project exports include ordinary workspace files up to 100 MiB each. If an export would include a larger ordinary file, the API rejects it with a structured error naming the relative path, so the archive cannot silently omit that file. Existing hidden, secret, symlink, and sanitized-config rules still apply. + ### Runtime Notes - Prefer a remote model API when experiments need the local GPU. Local model inference and long attention caches compete with training and evaluation for memory. diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 8c747e3..59f6246 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -683,7 +683,7 @@ Known domain failures: `400` NOT_FILE; `403` PATH_ESCAPE; `404` NOT_FOUND or MIS #### `POST /api/projects/{ident}/export` -ZIP contains forest-project.json and eligible workspace files. Optional paths selects included file prefixes, not a smaller graph/resource/run manifest. Export omits private graph history and machine fields while preserving the nonsecret project provider ID; credentials remain excluded. Large individual files (>100 MiB), symlinks and hidden paths are skipped. Do not treat the archive as an encrypted secret store. +ZIP contains forest-project.json and eligible workspace files. Optional paths selects included file prefixes, not a smaller graph/resource/run manifest. Export omits private graph history and machine fields while preserving the nonsecret project provider ID; credentials remain excluded. An included ordinary file above 100 MiB is rejected with a structured 413 EXPORT_FILE_TOO_LARGE error naming its relative path; symlinks and hidden paths are skipped. Do not treat the archive as an encrypted secret store. | Parameter | Location | Type | Required | Default / description | | --- | --- | --- | --- | --- | @@ -693,7 +693,7 @@ Body: `application/json`: `ProjectExportRequest`; optional. Success `200`: `application/zip`: `string`. -Known domain failures: `404` NOT_FOUND. +Known domain failures: `404` NOT_FOUND; `413` EXPORT_FILE_TOO_LARGE. #### `DELETE /api/projects/{ident}/file` diff --git a/packages/contracts/api.generated.ts b/packages/contracts/api.generated.ts index 7ae1706..4a5d1a2 100644 --- a/packages/contracts/api.generated.ts +++ b/packages/contracts/api.generated.ts @@ -395,7 +395,7 @@ export interface paths { get: { parameters: { path: { "ident": string; }; query?: never; header?: { "Last-Event-ID"?: string; }; cookie?: never; }; requestBody?: never; responses: { "200": { headers: { [name: string]: unknown }; content: { "text/event-stream": string; }; }; "401": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "403": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "404": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "422": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "500": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; }; }; }; "/api/projects/{ident}/export": { - post: { parameters: { path: { "ident": string; }; query?: never; header?: never; cookie?: never; }; requestBody?: { content: { "application/json": components["schemas"]["ProjectExportRequest"]; }; }; responses: { "200": { headers: { [name: string]: unknown }; content: { "application/zip": Blob; }; }; "401": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "403": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "404": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "422": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "500": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; }; }; + post: { parameters: { path: { "ident": string; }; query?: never; header?: never; cookie?: never; }; requestBody?: { content: { "application/json": components["schemas"]["ProjectExportRequest"]; }; }; responses: { "200": { headers: { [name: string]: unknown }; content: { "application/zip": Blob; }; }; "401": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "403": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "404": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "413": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "422": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "500": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; }; }; }; "/api/projects/{ident}/file": { delete: { parameters: { path: { "ident": string; }; query: { "path": string; }; header?: never; cookie?: never; }; requestBody?: never; responses: { "200": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["Deleted"]; }; }; "400": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "401": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "403": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "404": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "422": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; "500": { headers: { [name: string]: unknown }; content: { "application/json": components["schemas"]["ApiError"]; }; }; }; }; diff --git a/packages/contracts/openapi.json b/packages/contracts/openapi.json index 4755d9b..f8c1fd9 100644 --- a/packages/contracts/openapi.json +++ b/packages/contracts/openapi.json @@ -17661,7 +17661,7 @@ }, "/api/projects/{ident}/export": { "post": { - "description": "ZIP contains forest-project.json and eligible workspace files. Optional paths selects included file prefixes, not a smaller graph/resource/run manifest. Export omits private graph history and machine fields while preserving the nonsecret project provider ID; credentials remain excluded. Large individual files (>100 MiB), symlinks and hidden paths are skipped. Do not treat the archive as an encrypted secret store.", + "description": "ZIP contains forest-project.json and eligible workspace files. Optional paths selects included file prefixes, not a smaller graph/resource/run manifest. Export omits private graph history and machine fields while preserving the nonsecret project provider ID; credentials remain excluded. An included ordinary file above 100 MiB is rejected with a structured 413 EXPORT_FILE_TOO_LARGE error naming its relative path; symlinks and hidden paths are skipped. Do not treat the archive as an encrypted secret store.", "operationId": "export_project_api_projects__ident__export_post", "parameters": [ { @@ -17725,6 +17725,16 @@ }, "description": "NOT_FOUND" }, + "413": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + }, + "description": "EXPORT_FILE_TOO_LARGE" + }, "422": { "content": { "application/json": { diff --git a/services/api/contracts.py b/services/api/contracts.py index 6c9d1fd..20a6237 100644 --- a/services/api/contracts.py +++ b/services/api/contracts.py @@ -634,7 +634,7 @@ def operation(method: str, path: str, tag: str, summary: str, description: str, operation("post", "/api/projects/{ident}/upload", "Files", "Upload a workspace file", "multipart/form-data field file; directory is a QUERY parameter, default uploads. Only the uploaded filename basename is used. Publish the file, increment its file revision (including identical-byte replacements), set user_import origin, mark consumers stale and return path/size/origin. Older editor expected_revision values then return REVISION_CONFLICT. Exceeding configured max_upload_mb preserves an existing destination and removes only the temporary file.", ref("FileUpload"), errors={403: "PATH_ESCAPE", 404: "NOT_FOUND", 409: "WORKSPACE_PENDING", 413: "UPLOAD_TOO_LARGE"}) operation("post", "/api/projects/{ident}/export", "Files", "Export a project backup ZIP", - "ZIP contains forest-project.json and eligible workspace files. Optional paths selects included file prefixes, not a smaller graph/resource/run manifest. Export omits private graph history and machine fields while preserving the nonsecret project provider ID; credentials remain excluded. Large individual files (>100 MiB), symlinks and hidden paths are skipped. Do not treat the archive as an encrypted secret store.", body="ProjectExportRequest", media=("application/zip",), errors={404: "NOT_FOUND"}) + "ZIP contains forest-project.json and eligible workspace files. Optional paths selects included file prefixes, not a smaller graph/resource/run manifest. Export omits private graph history and machine fields while preserving the nonsecret project provider ID; credentials remain excluded. An included ordinary file above 100 MiB is rejected with a structured 413 EXPORT_FILE_TOO_LARGE error naming its relative path; symlinks and hidden paths are skipped. Do not treat the archive as an encrypted secret store.", body="ProjectExportRequest", media=("application/zip",), errors={404: "NOT_FOUND", 413: "EXPORT_FILE_TOO_LARGE"}) operation("post", "/api/projects/import", "Files", "Import a project backup ZIP", "multipart/form-data file with forest-project-v1 manifest. Validate graph and archive paths, reject symlinks/path traversal and >500 MiB decompressed archives. Create a new project with remapped graph/resource identifiers; preserve its selected provider when that ID exists in the current workspace. If the saved provider is unavailable, mark provider_selection_required and prevent provider-backed runs from silently falling back until the owner edits project settings. Active imported work is interrupted and execution is not replayed.", ref("Project"), errors={400: "INVALID_ARCHIVE, UNSAFE_ARCHIVE or graph validation errors", 413: "UPLOAD_TOO_LARGE or ARCHIVE_TOO_LARGE"}) diff --git a/services/api/files.py b/services/api/files.py index 9ddc35f..04d7563 100644 --- a/services/api/files.py +++ b/services/api/files.py @@ -36,6 +36,7 @@ # short-lived and process-local; the API currently runs as one Uvicorn worker. _FILE_CURSOR_TTL=600 _FILE_CURSOR_LIMIT=256 +_EXPORT_MAX_FILE_BYTES=100*1024*1024 _FILE_CURSOR_LOCK=threading.RLock() _FILE_CURSORS=OrderedDict() @@ -542,7 +543,10 @@ def project_export(s,p,selection=None): rel=str(f.relative_to(root)) if f.is_file() and not f.is_symlink() and f.name not in ('config.json','secrets.json','.env','owner-token') and not any(part.startswith('.') for part in f.relative_to(root).parts): if selection and not any(rel.startswith(x) for x in selection): continue - if f.stat().st_size<=100*1024*1024: z.write(f,'files/'+rel) + size=f.stat().st_size + if size>_EXPORT_MAX_FILE_BYTES: + error('EXPORT_FILE_TOO_LARGE',f'Cannot export {rel}: file exceeds 100 MiB limit',413) + z.write(f,'files/'+rel) elif f.is_file() and not f.is_symlink() and f.name=='config.json' and not any(part.startswith('.') for part in f.relative_to(root).parts): if selection and not any(rel.startswith(x) for x in selection): continue try: diff --git a/tests/test_api.py b/tests/test_api.py index 825959a..58a9f13 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -487,6 +487,59 @@ def case_export_import_reference_roundtrip(client, app): assert ok(client.get(f"/api/runs/{new_runs[0]['id']}/output"))["text"] == "actual saved output\n" +def case_export_rejects_oversized_files_and_keeps_boundary_roundtrip(client, app): + p = create(client) + root = Path(os.environ["FOREST_DATA_DIR"]) / "projects" / p["id"] + (root / "keep.txt").write_text("retained export file\n") + + oversized = root / "research" / "oversized-artifact.bin" + oversized.parent.mkdir(parents=True) + with oversized.open("wb") as stream: + stream.truncate(100 * 1024 * 1024 + 1) + + rejected = client.post(f"/api/projects/{p['id']}/export", json={}) + assert rejected.status_code == 413 + detail = rejected.json()["detail"] + assert detail["code"] == "EXPORT_FILE_TOO_LARGE" + assert "research/oversized-artifact.bin" in detail["message"] + assert "100 MiB" in detail["message"] + + selected = client.post(f"/api/projects/{p['id']}/export", json={"paths": ["keep.txt"]}) + assert selected.status_code == 200, selected.text + with zipfile.ZipFile(io.BytesIO(selected.content)) as archive: + assert archive.read("files/keep.txt") == b"retained export file\n" + assert "files/research/oversized-artifact.bin" not in archive.namelist() + + oversized.unlink() + boundary = root / "boundary-artifact.bin" + with boundary.open("wb") as stream: + stream.truncate(100 * 1024 * 1024) + stream.seek(0) + stream.write(b"boundary-start") + stream.seek(-len(b"boundary-end"), os.SEEK_END) + stream.write(b"boundary-end") + exported = client.post(f"/api/projects/{p['id']}/export", json={}) + assert exported.status_code == 200, exported.text + with zipfile.ZipFile(io.BytesIO(exported.content)) as archive: + assert archive.getinfo("files/boundary-artifact.bin").file_size == 100 * 1024 * 1024 + boundary_member = archive.open("files/boundary-artifact.bin") + assert boundary_member.read(len(b"boundary-start")) == b"boundary-start" + boundary_member.seek(-len(b"boundary-end"), os.SEEK_END) + assert boundary_member.read() == b"boundary-end" + assert archive.testzip() is None + restored = client.post("/api/projects/import", files={ + "file": ("boundary-project.zip", exported.content, "application/zip"), + }) + assert restored.status_code == 200, restored.text + restored_root = Path(os.environ["FOREST_DATA_DIR"]) / "projects" / restored.json()["id"] + assert (restored_root / "boundary-artifact.bin").stat().st_size == 100 * 1024 * 1024 + with (restored_root / "boundary-artifact.bin").open("rb") as restored_boundary: + assert restored_boundary.read(len(b"boundary-start")) == b"boundary-start" + restored_boundary.seek(-len(b"boundary-end"), os.SEEK_END) + assert restored_boundary.read() == b"boundary-end" + assert (restored_root / "keep.txt").read_text() == "retained export file\n" + + def case_project_provider_roundtrip_requires_explicit_resolution_when_missing(client, app): provider = ok(client.post('/api/providers', json={ 'name': 'Project archive provider', 'kind': 'openai', diff --git a/tests/test_api_contracts.py b/tests/test_api_contracts.py index a87f541..3a4aedd 100644 --- a/tests/test_api_contracts.py +++ b/tests/test_api_contracts.py @@ -178,6 +178,8 @@ def case_documented_transports(client, app): ("/api/library/export-bibtex", "get", "application/x-bibtex"), ("/api/papers/{ident}/export", "post", "application/pdf")): assert media in paths[endpoint][method]["responses"]["200"]["content"] + export_contract = paths["/api/projects/{ident}/export"]["post"] + assert export_contract["responses"]["413"]["description"] == "EXPORT_FILE_TOO_LARGE" assert "/ws/projects/{ident}/terminal" in json.dumps(document["x-forest-websockets"])