diff --git a/Makefile b/Makefile index ec43f259c..f8a61f9f5 100644 --- a/Makefile +++ b/Makefile @@ -214,9 +214,12 @@ evaluate: ################################################## # OpenAPI and model generation -.PHONY: merge-openapi generate-models generate-frontend-sdk +.PHONY: openapi-check merge-openapi generate-models generate-frontend-sdk +openapi-check: + @npx --yes @redocly/cli bundle aperag/api/openapi.yaml --output /tmp/openapi-check-bundle.yaml && rm -f /tmp/openapi-check-bundle.yaml + merge-openapi: - @cd aperag && redocly bundle ./api/openapi.yaml > ./api/openapi.merged.yaml + @cd aperag && npx --yes @redocly/cli bundle ./api/openapi.yaml > ./api/openapi.merged.yaml generate-models: merge-openapi @datamodel-codegen \ diff --git a/PR_DESCRIPTION_1096.md b/PR_DESCRIPTION_1096.md new file mode 100644 index 000000000..dd1742980 --- /dev/null +++ b/PR_DESCRIPTION_1096.md @@ -0,0 +1,17 @@ +# Summary +This PR updates marketplace collection graph pages to use marketplace-safe graph APIs and improves graph page layout behavior on shorter screens. + +## What changed +- Render the marketplace collection `/graph` page with the new graph-hybrid view. +- Add read-only marketplace graph endpoints for `embedding-map` and `entity-search` so published marketplace collections do not call workspace-only graph APIs. +- Let `CollectionGraphHybrid` switch between workspace and marketplace data sources. +- Give graph pages a 720px minimum graph area so shorter screens can scroll instead of compressing the canvas. + +## Why +Marketplace collection graph pages should not depend on workspace-only APIs. This change introduces marketplace-safe read-only graph endpoints and updates frontend routing/data-source behavior so graph rendering works correctly for published collections while preserving workspace behavior. + +## Validation +- `make openapi-check` +- `uv run ruff check aperag/domains/marketplace/api/routes.py` +- `yarn type-check` +- `yarn lint` (passes with pre-existing unrelated warnings) diff --git a/PR_DESCRIPTION_1963.md b/PR_DESCRIPTION_1963.md new file mode 100644 index 000000000..568cc6879 --- /dev/null +++ b/PR_DESCRIPTION_1963.md @@ -0,0 +1,46 @@ +# Summary +This PR addresses issue #1963 where `PUT /api/v2/collections/{id}` returns stable `500 DATABASE_ERROR` when updating an existing collection config in apemind POC (SG) / SG evaluation environments. + +## Problem +- Updating an existing `type=document` collection via `PUT /api/v2/collections/{collection_id}` consistently fails with: + - `{"success":false,"error_code":"DATABASE_ERROR","code":1050,"message":"数据库出现错误，请稍后重试。"}` +- Retry does not recover; failure is stable. + +## Reproduction +1. Create a `type=document` collection (POST path succeeds). +2. Update config with `PUT /api/v2/collections/{collection_id}`: + - Reproduces when changing only `enable_vector` / `fulltext` / `embedding`. + - Also reproduces when enabling knowledge graph (`enable_knowledge_graph=true`). +3. Observe stable `500 DATABASE_ERROR`. + +## Impact +- Existing collections cannot be reconfigured. +- Typical operation "enable knowledge graph after collection creation" is blocked. +- Current workaround is delete + recreate collection, which is high cost and may lose built index state. + +## Isolation Findings +- `POST /api/v2/collections` (CREATE): normal. +- `GET` paths: normal. +- Only `PUT` update path is failing. +- Failure is not knowledge-graph specific. + +## Scope in this PR +- Triage and root-cause analysis for collection update failure in update path. +- Confirm ownership boundary between KB domain collection update service and DB layer. +- Implement and validate fix for `DATABASE_ERROR` in update flow. + +## Validation Plan +- Reproduce on apemind POC (SG) with an existing collection. +- Verify PUT update succeeds for: + - Non-graph config-only changes (`enable_vector` / `fulltext` / `embedding`). + - Graph enablement path (`enable_knowledge_graph=true`). +- Regression check: + - CREATE remains normal. + - GET remains normal. + - No regression in collection update behavior across existing test fixtures. + +## Context +- Environment where issue was found: apemind POC (SG). +- Discovery date: 2026-07-01. +- Reporter: @cuiwenbo (崔文博), during `task feat: chat #17`. +- Tracking issue: #1963. diff --git a/README-zh.md b/README-zh.md index 47cf6e120..a15a88019 100644 --- a/README-zh.md +++ b/README-zh.md @@ -6,8 +6,6 @@ ApeRAG 是一个生产级 RAG（检索增强生成）平台，结合了图 RAG、向量搜索、全文搜索和先进的 AI 智能体。构建具有混合检索、多模态文档处理、智能代理和企业级管理功能的复杂 AI 应用程序。 -**🚀 [在线体验 ApeRAG](https://rag.apecloud.com/)** - 通过我们的托管演示体验完整的平台功能 - ApeRAG 是你构建自己的知识图谱、进行上下文工程以及部署能够自主搜索和推理知识库的智能 AI 代理的最佳选择。 [Read English Documentation](README.md) @@ -51,7 +49,7 @@ ApeRAG 支持 [MCP（模型上下文协议）](https://modelcontextprotocol.io/) { "mcpServers": { "aperag-mcp": { - "url": "https://rag.apecloud.com/mcp/", + "url": "http://localhost:8000/mcp/", "headers": { "Authorization": "Bearer your-api-key-here" } @@ -64,7 +62,7 @@ ApeRAG 支持 [MCP（模型上下文协议）](https://modelcontextprotocol.io/) 1. **HTTP Authorization 头**（推荐）：`Authorization: Bearer your-api-key` 2. **环境变量**（备用）：`APERAG_API_KEY=your-api-key` -**重要提示**：将 `https://rag.apecloud.com` 替换为您实际的 ApeRAG API 地址，将 `your-api-key-here` 替换为 ApeRAG 设置中的有效 API 密钥。 +**重要提示**：若为非本机部署，请将示例 URL 换成实际 API 源地址对应的 MCP 路径（如 `https://<你的域名>/mcp/`）。将 `your-api-key-here` 替换为 ApeRAG 设置中的有效 API 密钥。 MCP 服务器提供： - **集合浏览**：列出和探索您的知识集合 diff --git a/README.md b/README.md index 532a66339..f95563fc1 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ # ApeRAG [![Trust Score](https://archestra.ai/mcp-catalog/api/badge/quality/apecloud/ApeRAG)](https://archestra.ai/mcp-catalog/apecloud__aperag) -**🚀 [Try ApeRAG Live Demo](https://rag.apecloud.com/)** - Experience the full platform capabilities with our hosted demo - - ![HarryPotterKG2.png](docs%2Fen-US%2Fimages%2FHarryPotterKG2.png) ![chat2.png](docs%2Fen-US%2Fimages%2Fchat2.png) @@ -52,7 +49,7 @@ ApeRAG supports [MCP (Model Context Protocol)](https://modelcontextprotocol.io/) { "mcpServers": { "aperag-mcp": { - "url": "https://rag.apecloud.com/mcp/", + "url": "http://localhost:8000/mcp/", "headers": { "Authorization": "Bearer your-api-key-here" } @@ -65,7 +62,7 @@ ApeRAG supports [MCP (Model Context Protocol)](https://modelcontextprotocol.io/) 1. **HTTP Authorization Header** (Recommended): `Authorization: Bearer your-api-key` 2. **Environment Variable** (Fallback): `APERAG_API_KEY=your-api-key` -**Important**: Replace `https://rag.apecloud.com` with your actual ApeRAG API URL and `your-api-key-here` with a valid API key from your ApeRAG settings. +**Important**: Use your deployed API origin if not local (e.g. `https://your-host/mcp/`). Replace `your-api-key-here` with a valid API key from your ApeRAG settings. The MCP server provides: - **Collection browsing**: List and explore your knowledge collections diff --git a/aperag/api/components/schemas/document.yaml b/aperag/api/components/schemas/document.yaml index 64cbe95b6..7683f8d13 100644 --- a/aperag/api/components/schemas/document.yaml +++ b/aperag/api/components/schemas/document.yaml @@ -251,3 +251,89 @@ confirmDocumentsResponse: required: - confirmed_count - failed_count + +fetchUrlRequest: + type: object + properties: + urls: + type: array + items: + type: string + format: uri + minItems: 1 + maxItems: 10 + description: List of URLs to fetch and import (max 10) + example: + - "https://example.com/article1" + - "https://example.com/article2" + required: + - urls + +fetchUrlResultItem: + type: object + properties: + url: + type: string + description: The source URL + fetch_status: + type: string + enum: + - success + - error + description: Whether the URL was fetched successfully + document_id: + type: string + description: ID of the created document (only present on success) + filename: + type: string + description: Filename of the created document (only present on success) + size: + type: integer + description: Size of the created document in bytes (only present on success) + status: + type: string + description: Document status (only present on success) + error: + type: string + description: Error message (only present on failure) + required: + - url + - fetch_status + +fetchUrlResponse: + type: object + properties: + results: + type: array + items: + $ref: '#/fetchUrlResultItem' + description: Results for each URL + total: + type: integer + description: Total number of URLs processed + succeeded: + type: integer + description: Number of URLs successfully fetched + failed: + type: integer + description: Number of URLs that failed + required: + - results + - total + - succeeded + - failed + +stagedDocumentsResponse: + type: object + properties: + documents: + type: array + items: + $ref: '#/uploadDocumentResponse' + description: List of staged (UPLOADED) documents awaiting confirmation + total: + type: integer + description: Total number of staged documents + required: + - documents + - total diff --git a/aperag/api/openapi.yaml b/aperag/api/openapi.yaml index 7e4becf08..573c4fb2a 100644 --- a/aperag/api/openapi.yaml +++ b/aperag/api/openapi.yaml @@ -73,6 +73,10 @@ paths: $ref: './paths/collections.yaml#/upload_document' /collections/{collection_id}/documents/confirm: $ref: './paths/collections.yaml#/confirm_documents' + /collections/{collection_id}/documents/fetch-url: + $ref: './paths/collections.yaml#/fetch_url_document' + /collections/{collection_id}/documents/staged: + $ref: './paths/collections.yaml#/list_staged_documents' /collections/{collection_id}/searches: $ref: './paths/collections.yaml#/searches' /collections/{collection_id}/searches/{search_id}: @@ -111,6 +115,10 @@ paths: $ref: './paths/collections.yaml#/graph_suggestion_action' /collections/{collection_id}/graphs/export/kg-eval: $ref: './paths/collections.yaml#/graph_export_kg_eval' + /collections/{collection_id}/graphs/embedding-map: + $ref: './paths/collections.yaml#/graph_embedding_map' + /collections/{collection_id}/graphs/entity-search: + $ref: './paths/collections.yaml#/graph_entity_search' /collections/{collection_id}/sharing: $ref: './paths/collections.yaml#/sharing' @@ -131,6 +139,10 @@ paths: $ref: './paths/marketplace.yaml#/marketplaceCollectionDocumentObject' /marketplace/collections/{collection_id}/graph: $ref: './paths/marketplace.yaml#/marketplaceCollectionGraph' + /marketplace/collections/{collection_id}/graph/embedding-map: + $ref: './paths/marketplace.yaml#/marketplaceCollectionGraphEmbeddingMap' + /marketplace/collections/{collection_id}/graph/entity-search: + $ref: './paths/marketplace.yaml#/marketplaceCollectionGraphEntitySearch' # apikeys /apikeys: diff --git a/aperag/api/paths/collections.yaml b/aperag/api/paths/collections.yaml index 402bc4ad4..724ee8b73 100644 --- a/aperag/api/paths/collections.yaml +++ b/aperag/api/paths/collections.yaml @@ -664,6 +664,98 @@ confirm_documents: schema: $ref: '../components/schemas/common.yaml#/failResponse' +fetch_url_document: + post: + summary: Fetch documents from URLs + description: | + Fetch web page content from one or more URLs and create UPLOADED documents. + Each URL is fetched using the web read service (JINA with Trafilatura fallback). + Successfully fetched URLs produce UPLOADED documents in the staging area, + identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing. + security: + - BearerAuth: [] + parameters: + - name: collection_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/schemas/document.yaml#/fetchUrlRequest' + examples: + single_url: + summary: Single URL + value: + urls: + - "https://example.com/article" + multiple_urls: + summary: Multiple URLs + value: + urls: + - "https://example.com/article1" + - "https://example.com/article2" + responses: + '200': + description: URL fetch completed (partial success is also 200) + content: + application/json: + schema: + $ref: '../components/schemas/document.yaml#/fetchUrlResponse' + '400': + description: Bad request - invalid URLs or too many URLs + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + +list_staged_documents: + get: + summary: List staged documents + description: Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation. + security: + - BearerAuth: [] + parameters: + - name: collection_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Staged documents retrieved successfully + content: + application/json: + schema: + $ref: '../components/schemas/document.yaml#/stagedDocumentsResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + searches: get: summary: Get search history @@ -1253,6 +1345,105 @@ graph_export_kg_eval: schema: $ref: '../components/schemas/common.yaml#/failResponse' +graph_embedding_map: + get: + summary: Get entity list for embedding map visualization + description: Returns entities with their degree counts for use in an embedding map scatter-plot visualization. + tags: + - graph + security: + - BearerAuth: [] + parameters: + - name: collection_id + in: path + required: true + schema: + type: string + description: Collection ID + - name: max_nodes + in: query + schema: + type: integer + minimum: 1 + maximum: 5000 + default: 500 + description: Maximum number of entities to return + responses: + '200': + description: Entity list for embedding map + content: + application/json: + schema: + type: object + properties: + entities: + type: array + items: + type: object + total: + type: integer + '401': + $ref: '../components/schemas/common.yaml#/failResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + +graph_entity_search: + get: + summary: Search entities in the knowledge graph + description: Full-text search on entity names within the collection's knowledge graph. + tags: + - graph + security: + - BearerAuth: [] + parameters: + - name: collection_id + in: path + required: true + schema: + type: string + description: Collection ID + - name: q + in: query + required: true + schema: + type: string + minLength: 1 + description: Search query + - name: max_results + in: query + schema: + type: integer + minimum: 1 + maximum: 500 + default: 50 + description: Maximum number of results to return + responses: + '200': + description: Matching entities + content: + application/json: + schema: + type: object + properties: + nodes: + type: array + items: + type: object + total: + type: integer + '401': + $ref: '../components/schemas/common.yaml#/failResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + sharing: get: summary: Get collection sharing status diff --git a/aperag/api/paths/marketplace.yaml b/aperag/api/paths/marketplace.yaml index 1deb8e0e1..a2f86d609 100644 --- a/aperag/api/paths/marketplace.yaml +++ b/aperag/api/paths/marketplace.yaml @@ -405,4 +405,107 @@ marketplaceCollectionGraph: schema: $ref: '../components/schemas/common.yaml#/failResponse' '500': - $ref: '../components/schemas/common.yaml#/failResponse' \ No newline at end of file + $ref: '../components/schemas/common.yaml#/failResponse' + +marketplaceCollectionGraphEmbeddingMap: + get: + summary: Get entity list for embedding map visualization (read-only, marketplace-safe) + description: Returns entities with degree counts for embedding map visualization. Uses the collection owner's credentials. + parameters: + - name: collection_id + in: path + required: true + description: Collection ID + schema: + type: string + - name: max_nodes + in: query + schema: + type: integer + minimum: 1 + maximum: 5000 + default: 500 + description: Maximum number of entities to return + responses: + '200': + description: Entity list for embedding map + content: + application/json: + schema: + type: object + properties: + entities: + type: array + items: + type: object + total: + type: integer + '403': + description: Access denied (need subscription) + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + '404': + description: Collection not found or not published + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + '500': + $ref: '../components/schemas/common.yaml#/failResponse' + +marketplaceCollectionGraphEntitySearch: + get: + summary: Search entities in the knowledge graph (read-only, marketplace-safe) + description: Full-text search on entity names. Uses the collection owner's credentials. + parameters: + - name: collection_id + in: path + required: true + description: Collection ID + schema: + type: string + - name: q + in: query + required: true + schema: + type: string + minLength: 1 + description: Search query + - name: max_results + in: query + schema: + type: integer + minimum: 1 + maximum: 500 + default: 50 + description: Maximum number of results + responses: + '200': + description: Matching entities + content: + application/json: + schema: + type: object + properties: + nodes: + type: array + items: + type: object + total: + type: integer + '403': + description: Access denied (need subscription) + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + '404': + description: Collection not found or not published + content: + application/json: + schema: + $ref: '../components/schemas/common.yaml#/failResponse' + '500': + $ref: '../components/schemas/common.yaml#/failResponse' diff --git a/aperag/app.py b/aperag/app.py index 73de22a2b..2cff5559b 100644 --- a/aperag/app.py +++ b/aperag/app.py @@ -35,9 +35,11 @@ from aperag.agent.agent_event_listener import agent_event_listener # noqa: E402 from aperag.agent.agent_session_manager_lifecycle import agent_session_manager_lifespan # noqa: E402 +from aperag.domains.marketplace.api.routes import router as marketplace_graph_router from aperag.exception_handlers import register_exception_handlers from aperag.llm.litellm_track import register_custom_llm_track from aperag.mcp import mcp_server +from aperag.middleware.latency import LatencyLoggingMiddleware from aperag.views.api_key import router as api_key_router from aperag.views.audit import router as audit_router from aperag.views.auth import router as auth_router @@ -86,6 +88,10 @@ async def combined_lifespan(app: FastAPI): # Register global exception handlers register_exception_handlers(app) +# Measure and log the wall-clock duration of every HTTP request. +# The middleware also adds an ``X-Response-Time`` header to each response. +app.add_middleware(LatencyLoggingMiddleware) + register_custom_llm_track() @@ -107,6 +113,7 @@ async def health_check(): app.include_router(graph_router, prefix="/api/v1") app.include_router(marketplace_router, prefix="/api/v1") # Add marketplace router app.include_router(marketplace_collections_router, prefix="/api/v1") # Add marketplace collections router +app.include_router(marketplace_graph_router, prefix="/api/v1") # Add marketplace graph router app.include_router(settings_router, prefix="/api/v1") app.include_router(prompts_router, prefix="/api/v1") # Add prompts router app.include_router(web_router, prefix="/api/v1") # Add web search router diff --git a/aperag/db/models.py b/aperag/db/models.py index e833611e2..691b3f704 100644 --- a/aperag/db/models.py +++ b/aperag/db/models.py @@ -841,6 +841,7 @@ class AuditLog(Base): request_id = Column(String(255), nullable=False, comment="Request ID for tracking") start_time = Column(BigInteger, nullable=False, comment="Request start time (milliseconds since epoch)") end_time = Column(BigInteger, nullable=True, comment="Request end time (milliseconds since epoch)") + duration_ms = Column(BigInteger, nullable=True, comment="Request duration in milliseconds (end_time - start_time)") gmt_created = Column(DateTime(timezone=True), nullable=False, default=utc_now, comment="Created time") # Index for better query performance @@ -854,6 +855,7 @@ class AuditLog(Base): Index("idx_audit_resource_id", "resource_id"), Index("idx_audit_request_id", "request_id"), Index("idx_audit_start_time", "start_time"), + Index("idx_audit_duration_ms", "duration_ms"), ) def __repr__(self): diff --git a/aperag/domains/__init__.py b/aperag/domains/__init__.py new file mode 100644 index 000000000..676ec3691 --- /dev/null +++ b/aperag/domains/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/aperag/domains/marketplace/__init__.py b/aperag/domains/marketplace/__init__.py new file mode 100644 index 000000000..676ec3691 --- /dev/null +++ b/aperag/domains/marketplace/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/aperag/domains/marketplace/api/__init__.py b/aperag/domains/marketplace/api/__init__.py new file mode 100644 index 000000000..676ec3691 --- /dev/null +++ b/aperag/domains/marketplace/api/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/aperag/domains/marketplace/api/routes.py b/aperag/domains/marketplace/api/routes.py new file mode 100644 index 000000000..ebfb98945 --- /dev/null +++ b/aperag/domains/marketplace/api/routes.py @@ -0,0 +1,84 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read-only marketplace graph endpoints for embedding-map and entity-search. + +These endpoints mirror workspace-only graph APIs but are safe for published +marketplace collections — they resolve to the collection owner's user_id so +that callers do not need to own the collection. +""" + +import logging + +from fastapi import APIRouter, Depends, HTTPException, Query + +from aperag.db.models import User +from aperag.exceptions import CollectionMarketplaceAccessDeniedError, CollectionNotPublishedError +from aperag.service.marketplace_collection_service import marketplace_collection_service +from aperag.views.auth import optional_user + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["graph"]) + + +@router.get("/marketplace/collections/{collection_id}/graph/embedding-map") +async def get_marketplace_graph_embedding_map( + collection_id: str, + max_nodes: int = Query(500, ge=1, le=5000), + user: User = Depends(optional_user), +): + """Get entity list for embedding map visualization (read-only, marketplace-safe).""" + from aperag.service.graph_service import graph_service + + try: + user_id = str(user.id) if user else "" + marketplace_info = await marketplace_collection_service._check_marketplace_access(user_id, collection_id) + owner_user_id = marketplace_info["owner_user_id"] + return await graph_service.get_embedding_map(str(owner_user_id), collection_id, max_nodes) + except CollectionNotPublishedError: + raise HTTPException(status_code=404, detail="Collection not found or not published") + except CollectionMarketplaceAccessDeniedError as e: + raise HTTPException(status_code=403, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error getting marketplace graph embedding map {collection_id}: {e}") + raise HTTPException(status_code=500, detail="Internal server error") + + +@router.get("/marketplace/collections/{collection_id}/graph/entity-search") +async def get_marketplace_graph_entity_search( + collection_id: str, + q: str = Query(..., min_length=1, description="Search query"), + max_results: int = Query(50, ge=1, le=500), + user: User = Depends(optional_user), +): + """Search entities in the knowledge graph by name (read-only, marketplace-safe).""" + from aperag.service.graph_service import graph_service + + try: + user_id = str(user.id) if user else "" + marketplace_info = await marketplace_collection_service._check_marketplace_access(user_id, collection_id) + owner_user_id = marketplace_info["owner_user_id"] + return await graph_service.search_entities(str(owner_user_id), collection_id, q, max_results) + except CollectionNotPublishedError: + raise HTTPException(status_code=404, detail="Collection not found or not published") + except CollectionMarketplaceAccessDeniedError as e: + raise HTTPException(status_code=403, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error searching marketplace graph entities {collection_id}: {e}") + raise HTTPException(status_code=500, detail="Internal server error") diff --git a/aperag/graph/lightrag/kg/neo4j_sync_impl.py b/aperag/graph/lightrag/kg/neo4j_sync_impl.py index 1c47fcf3e..5df5d1d60 100644 --- a/aperag/graph/lightrag/kg/neo4j_sync_impl.py +++ b/aperag/graph/lightrag/kg/neo4j_sync_impl.py @@ -460,6 +460,8 @@ def _sync_get_knowledge_graph(): result = KnowledgeGraph() seen_nodes = set() seen_edges = set() + # Map Neo4j internal id -> entity_id for edges (unified semantics: source/target = entity_id) + internal_id_to_entity_id: dict[int, str] = {} with Neo4jSyncConnectionManager.get_session(database=self._DATABASE) as session: if node_label == "*": @@ -476,18 +478,22 @@ def _sync_get_knowledge_graph(): for record in node_results: node = record["n"] - node_id = node.id - if node_id not in seen_nodes: + internal_id = node.id + if internal_id not in seen_nodes: + entity_id = node.get("entity_id") + node_id = str(entity_id) if entity_id is not None else f"{internal_id}" + internal_id_to_entity_id[internal_id] = node_id + entity_type = node.get("entity_type") result.nodes.append( KnowledgeGraphNode( - id=f"{node_id}", - labels=[node.get("entity_id")], + id=node_id, + labels=[entity_type] if entity_type else [node_id], properties=dict(node), ) ) - seen_nodes.add(node_id) + seen_nodes.add(internal_id) - # Get edges between these nodes + # Get edges between these nodes; source/target must be entity_id edge_query = """ MATCH (a)-[r]-(b) WHERE id(a) IN $node_ids AND id(b) IN $node_ids @@ -499,15 +505,18 @@ def _sync_get_knowledge_graph(): rel = record["r"] edge_id = rel.id if edge_id not in seen_edges: - result.edges.append( - KnowledgeGraphEdge( - id=f"{edge_id}", - type=rel.type, - source=f"{record['a'].id}", - target=f"{record['b'].id}", - properties=dict(rel), + src_entity = internal_id_to_entity_id.get(record["a"].id) + tgt_entity = internal_id_to_entity_id.get(record["b"].id) + if src_entity is not None and tgt_entity is not None: + result.edges.append( + KnowledgeGraphEdge( + id=f"{edge_id}", + type=rel.type, + source=src_entity, + target=tgt_entity, + properties=dict(rel), + ) ) - ) seen_edges.add(edge_id) else: # BFS from specific node @@ -527,30 +536,37 @@ def _sync_get_knowledge_graph(): for record in results: if record["nodes"]: for node in record["nodes"]: - node_id = node.id - if node_id not in seen_nodes: + internal_id = node.id + if internal_id not in seen_nodes: + entity_id = node.get("entity_id") + node_id = str(entity_id) if entity_id is not None else f"{internal_id}" + internal_id_to_entity_id[internal_id] = node_id + entity_type = node.get("entity_type") result.nodes.append( KnowledgeGraphNode( - id=f"{node_id}", - labels=[node.get("entity_id")], + id=node_id, + labels=[entity_type] if entity_type else [node_id], properties=dict(node), ) ) - seen_nodes.add(node_id) + seen_nodes.add(internal_id) if record["rels"]: for rel in record["rels"]: edge_id = rel.id if edge_id not in seen_edges: - result.edges.append( - KnowledgeGraphEdge( - id=f"{edge_id}", - type=rel.type, - source=f"{rel.start_node.id}", - target=f"{rel.end_node.id}", - properties=dict(rel), + src_entity = internal_id_to_entity_id.get(rel.start_node.id) + tgt_entity = internal_id_to_entity_id.get(rel.end_node.id) + if src_entity is not None and tgt_entity is not None: + result.edges.append( + KnowledgeGraphEdge( + id=f"{edge_id}", + type=rel.type, + source=src_entity, + target=tgt_entity, + properties=dict(rel), + ) ) - ) seen_edges.add(edge_id) logger.info(f"Retrieved subgraph with {len(result.nodes)} nodes and {len(result.edges)} edges") diff --git a/aperag/graph/lightrag/kg/pg_ops_sync_graph_storage.py b/aperag/graph/lightrag/kg/pg_ops_sync_graph_storage.py index 1f6c6bf2b..4193399f2 100644 --- a/aperag/graph/lightrag/kg/pg_ops_sync_graph_storage.py +++ b/aperag/graph/lightrag/kg/pg_ops_sync_graph_storage.py @@ -316,7 +316,7 @@ def _sync_get_knowledge_graph(): nodes_data = db_ops.get_graph_nodes_batch(self.workspace, matching_labels) for entity_id, node_data in nodes_data.items(): - # Assemble properties from individual fields + # Unified semantics: id=entity_id, labels=[entity_type] or [entity_id], edges by entity_id properties = { "entity_id": node_data["entity_id"], "entity_type": node_data.get("entity_type"), @@ -331,10 +331,11 @@ def _sync_get_knowledge_graph(): # Remove None values for cleaner output properties = {k: v for k, v in properties.items() if v is not None} + entity_type = node_data.get("entity_type") result.nodes.append( KnowledgeGraphNode( id=entity_id, - labels=[node_data.get("entity_type", entity_id)], + labels=[entity_type] if entity_type else [entity_id], properties=properties, ) ) diff --git a/aperag/graph/lightrag/types.py b/aperag/graph/lightrag/types.py index 46f7c7319..d4bb1c1ec 100644 --- a/aperag/graph/lightrag/types.py +++ b/aperag/graph/lightrag/types.py @@ -44,16 +44,28 @@ class GPTKeywordExtractionFormat(BaseModel): class KnowledgeGraphNode(BaseModel): + """ + Unified semantics for graph storage and API: + - id: entity_id (business identifier). Used for display, node identity, and edge source/target. + Must be stable and unique per entity; frontend uses it as node label and for merge/APIs. + - labels: optional list of semantic labels, e.g. [entity_type] for categorization/filtering. + - properties: entity_id, entity_type, description, source_id, file_path, entity_name, etc. + """ + id: str labels: list[str] properties: dict[str, Any] # anything else goes here class KnowledgeGraphEdge(BaseModel): + """ + source/target must be the same as KnowledgeGraphNode.id (i.e. entity_id) for correct linking. + """ + id: str type: Optional[str] - source: str # id of source node - target: str # id of target node + source: str # entity_id of source node + target: str # entity_id of target node properties: dict[str, Any] # anything else goes here diff --git a/aperag/middleware/__init__.py b/aperag/middleware/__init__.py new file mode 100644 index 000000000..676ec3691 --- /dev/null +++ b/aperag/middleware/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/aperag/middleware/latency.py b/aperag/middleware/latency.py new file mode 100644 index 000000000..b27e34c44 --- /dev/null +++ b/aperag/middleware/latency.py @@ -0,0 +1,87 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Latency logging middleware. + +Measures wall-clock time for every HTTP request and emits a structured +``INFO`` log line. This gives operators a lightweight, always-on view of +API performance without requiring an external tracing back-end. + +Log format (one line per request):: + + INFO aperag.middleware.latency GET /api/v1/bots 200 42ms + +The ``X-Response-Time`` response header is also set so browser DevTools and +upstream proxies can surface the latency directly. +""" + +import logging +import time + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response +from starlette.types import ASGIApp + +logger = logging.getLogger(__name__) + +# Paths that generate a lot of noise but carry no useful perf signal. +_SKIP_PATHS = frozenset(["/health", "/docs", "/openapi.json", "/redoc"]) + + +class LatencyLoggingMiddleware(BaseHTTPMiddleware): + """ASGI middleware that measures and logs request latency. + + For every request it: + + 1. Records the wall-clock start time (``time.perf_counter``). + 2. Passes control to the next handler. + 3. Records the end time and computes ``duration_ms``. + 4. Emits an ``INFO`` log with ``method``, ``path``, ``status_code``, and + ``duration_ms``. + 5. Attaches ``X-Response-Time: ms`` to the response so that + upstream load balancers and browser DevTools surface latency directly. + + Paths listed in ``_SKIP_PATHS`` (e.g. ``/health``) are processed but + logged at ``DEBUG`` level to avoid flooding production logs. + """ + + def __init__(self, app: ASGIApp) -> None: + super().__init__(app) + + async def dispatch(self, request: Request, call_next) -> Response: + start = time.perf_counter() + status_code = 500 # default in case call_next raises + response = None + + try: + response = await call_next(request) + status_code = response.status_code + finally: + duration_ms = int((time.perf_counter() - start) * 1000) + path = request.url.path + + msg = "%s %s %d %dms", request.method, path, status_code, duration_ms + if path in _SKIP_PATHS: + logger.debug(*msg) + else: + logger.info(*msg) + + # Expose latency to the caller via a response header. + # Guard against the (rare) case where call_next raises before + # returning a response object. + if response is not None: + response.headers["X-Response-Time"] = f"{duration_ms}ms" + + return response diff --git a/aperag/migration/versions/20251001120000-add_duration_ms_to_audit_log.py b/aperag/migration/versions/20251001120000-add_duration_ms_to_audit_log.py new file mode 100644 index 000000000..bf6bf5a83 --- /dev/null +++ b/aperag/migration/versions/20251001120000-add_duration_ms_to_audit_log.py @@ -0,0 +1,45 @@ +"""add duration_ms to audit_log + +Revision ID: a1b2c3d4e5f6 +Revises: ef8cf2222205 +Create Date: 2025-10-01 12:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: Union[str, None] = "ef8cf2222205" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add duration_ms column and its index to audit_log. + + The column stores the pre-computed request duration (end_time - start_time) + in milliseconds so it can be sorted and filtered efficiently without + requiring a computed expression index. + + Existing rows will have duration_ms = NULL. The application back-fills the + value on-the-fly when it lists audit logs, so no data migration is needed. + """ + op.add_column( + "audit_log", + sa.Column( + "duration_ms", + sa.BigInteger(), + nullable=True, + comment="Request duration in milliseconds (end_time - start_time)", + ), + ) + op.create_index("idx_audit_duration_ms", "audit_log", ["duration_ms"]) + + +def downgrade() -> None: + op.drop_index("idx_audit_duration_ms", table_name="audit_log") + op.drop_column("audit_log", "duration_ms") diff --git a/aperag/schema/view_models.py b/aperag/schema/view_models.py index 49b9b4bb2..2ade80660 100644 --- a/aperag/schema/view_models.py +++ b/aperag/schema/view_models.py @@ -14,14 +14,23 @@ # generated by datamodel-codegen: # filename: openapi.merged.yaml -# timestamp: 2026-03-04T08:41:38+00:00 +# timestamp: 2026-03-09T07:45:40+00:00 from __future__ import annotations from datetime import datetime from typing import Any, Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, EmailStr, Field, RootModel, confloat, conint +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + EmailStr, + Field, + RootModel, + confloat, + conint, +) class ModelSpec(BaseModel): @@ -874,6 +883,51 @@ class ConfirmDocumentsResponse(BaseModel): ) +class FetchUrlRequest(BaseModel): + urls: list[AnyUrl] = Field( + ..., + description='List of URLs to fetch and import (max 10)', + examples=[['https://example.com/article1', 'https://example.com/article2']], + ) + + +class FetchUrlResultItem(BaseModel): + url: str = Field(..., description='The source URL') + fetch_status: Literal['success', 'error'] = Field( + ..., description='Whether the URL was fetched successfully' + ) + document_id: Optional[str] = Field( + None, description='ID of the created document (only present on success)' + ) + filename: Optional[str] = Field( + None, description='Filename of the created document (only present on success)' + ) + size: Optional[int] = Field( + None, + description='Size of the created document in bytes (only present on success)', + ) + status: Optional[str] = Field( + None, description='Document status (only present on success)' + ) + error: Optional[str] = Field( + None, description='Error message (only present on failure)' + ) + + +class FetchUrlResponse(BaseModel): + results: list[FetchUrlResultItem] = Field(..., description='Results for each URL') + total: int = Field(..., description='Total number of URLs processed') + succeeded: int = Field(..., description='Number of URLs successfully fetched') + failed: int = Field(..., description='Number of URLs that failed') + + +class StagedDocumentsResponse(BaseModel): + documents: list[UploadDocumentResponse] = Field( + ..., description='List of staged (UPLOADED) documents awaiting confirmation' + ) + total: int = Field(..., description='Total number of staged documents') + + class VectorSearchParams(BaseModel): topk: Optional[int] = Field(None, description='Top K results') similarity: Optional[confloat(ge=0.0, le=1.0)] = Field( diff --git a/aperag/service/audit_service.py b/aperag/service/audit_service.py index 00befd394..579842e7d 100644 --- a/aperag/service/audit_service.py +++ b/aperag/service/audit_service.py @@ -20,8 +20,10 @@ from typing import Any, Dict, Optional from sqlalchemy import and_, desc, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import sessionmaker -from aperag.config import get_async_session +from aperag.config import async_engine from aperag.db.models import AuditLog, AuditResource logger = logging.getLogger(__name__) @@ -116,6 +118,16 @@ def extract_resource_id_from_path(self, path: str, resource_type: AuditResource) return None + def _make_session(self) -> AsyncSession: + """Create a short-lived async session for write operations. + + Using a dedicated factory here means we open the session only for the + duration of the DB write, avoiding the ``async for ... break`` generator + antipattern and making the lifecycle explicit. + """ + factory = sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False) + return factory() + async def log_audit( self, user_id: Optional[str], @@ -134,12 +146,37 @@ async def log_audit( user_agent: Optional[str] = None, request_id: Optional[str] = None, ): - """Log an audit entry""" + """Log an audit entry and persist it to the database. + + All expensive work (serialization, duration calculation) is done + *before* the session is opened so the connection is held for the + minimum time possible. + """ if not self.enabled: return try: - # Create audit log entry + # Compute duration before touching the DB. + duration_ms: Optional[int] = None + if start_time is not None and end_time is not None: + duration_ms = end_time - start_time + + # Serialize request/response data outside the session. + serialized_request = self._safe_json_serialize(request_data) + serialized_response = self._safe_json_serialize(response_data) + + # Emit a structured log line so operators can tail logs without + # querying the database. + logger.info( + "audit %s %s %s status=%s duration_ms=%s", + http_method, + path, + api_name, + status_code, + duration_ms, + ) + + # Build the ORM object outside the session — no DB access needed. audit_log = AuditLog( id=str(uuid.uuid4()), user_id=user_id, @@ -151,24 +188,19 @@ async def log_audit( status_code=status_code, start_time=start_time, end_time=end_time, - request_data=self._safe_json_serialize(request_data), - response_data=self._safe_json_serialize(response_data), + duration_ms=duration_ms, + request_data=serialized_request, + response_data=serialized_response, error_message=error_message, ip_address=ip_address, user_agent=user_agent, request_id=request_id or str(uuid.uuid4()), ) - # Save to database with proper session management - async def _save_audit_log(session): + # Open the session only for the DB write; close it immediately after. + async with self._make_session() as session: session.add(audit_log) await session.commit() - return audit_log - - # Use get_async_session with proper session management - async for session in get_async_session(): - await _save_audit_log(session) - break # Only process one session except Exception as e: logger.error(f"Failed to log audit: {e}") @@ -193,7 +225,7 @@ async def list_audit_logs( # Define sort field mapping sort_mapping = { "created": AuditLog.gmt_created, - "duration": AuditLog.end_time - AuditLog.start_time, # Calculated field + "duration": AuditLog.duration_ms, # Stored column — sortable without expression "status_code": AuditLog.status_code, "api_name": AuditLog.api_name, } @@ -246,12 +278,13 @@ async def _list_audit_logs(session): return items, total - # Execute query with proper session management + # Execute query with proper session management. + # Open the session, execute the lightweight query, and close it before + # any post-processing so the connection is returned to the pool quickly. audit_logs = None total = 0 - async for session in get_async_session(): + async with self._make_session() as session: audit_logs, total = await _list_audit_logs(session) - break # Only process one session # Post-process audit logs outside of session to avoid long session occupation processed_logs = [] @@ -270,11 +303,10 @@ async def _list_audit_logs(session): else: log.resource_id = None - # Calculate duration if both times are available - if log.start_time and log.end_time: + # duration_ms is now stored on the row; fall back to computing + # it on the fly for rows written before this migration. + if log.duration_ms is None and log.start_time and log.end_time: log.duration_ms = log.end_time - log.start_time - else: - log.duration_ms = None processed_logs.append(log) diff --git a/aperag/service/document_service.py b/aperag/service/document_service.py index cd96c640d..a7f8c3112 100644 --- a/aperag/service/document_service.py +++ b/aperag/service/document_service.py @@ -1326,6 +1326,155 @@ async def _confirm_documents_atomically(session): confirmed_count=confirmed_count, failed_count=failed_count, failed_documents=failed_documents ) + async def get_staged_documents(self, user_id: str, collection_id: str) -> view_models.StagedDocumentsResponse: + """Return all UPLOADED (staged) documents for the collection, ordered newest-first.""" + collection = await self._validate_collection(user_id, collection_id) + + async def _query(session: AsyncSession): + stmt = ( + select(db_models.Document) + .where( + db_models.Document.user == user_id, + db_models.Document.collection_id == collection.id, + db_models.Document.status == db_models.DocumentStatus.UPLOADED, + db_models.Document.gmt_deleted.is_(None), + ) + .order_by(db_models.Document.gmt_created.asc()) + ) + result = await session.execute(stmt) + return result.scalars().all() + + docs = await self.db_ops.execute_with_transaction(_query) + return view_models.StagedDocumentsResponse( + documents=[ + view_models.UploadDocumentResponse( + document_id=doc.id, + filename=doc.name, + size=doc.size or 0, + status=doc.status, + ) + for doc in docs + ], + total=len(docs), + ) + + async def fetch_url_documents(self, user_id: str, collection_id: str, urls: list) -> view_models.FetchUrlResponse: + """ + Fetch web page content from URLs and create UPLOADED documents. + + For each URL, uses the web read service (JINA with Trafilatura fallback) to + retrieve the page content as Markdown. The result is wrapped as a virtual + UploadFile and passed to upload_document(), so the resulting documents are + identical to file uploads and go through the same two-phase commit flow. + """ + import io + import re + from urllib.parse import urlparse + + from fastapi import UploadFile + from starlette.datastructures import Headers + + from aperag.db.ops import async_db_ops as _db_ops + from aperag.schema.view_models import WebReadRequest + from aperag.websearch.reader.reader_service import ReaderService + + # Validate URL count + if len(urls) > 10: + raise HTTPException(status_code=400, detail="Too many URLs: maximum 10 URLs per request") + + url_strings = [str(u) for u in urls] + + # Determine which reader to use based on user's JINA API key + jina_api_key = await _db_ops.query_provider_api_key("jina", user_id=user_id, need_public=True) + + web_read_request = WebReadRequest(url_list=url_strings, timeout=30) + + try: + if jina_api_key: + async with ReaderService(provider_name="jina", provider_config={"api_key": jina_api_key}) as svc: + web_response = await svc.read(web_read_request) + # Check if JINA returned any successes; fallback if not + if not any(r.status == "success" for r in web_response.results): + async with ReaderService(provider_name="trafilatura") as svc: + web_response = await svc.read(web_read_request) + else: + async with ReaderService(provider_name="trafilatura") as svc: + web_response = await svc.read(web_read_request) + except Exception as e: + logger.error(f"Web read service failed: {e}") + # Return all URLs as failed + results = [ + view_models.FetchUrlResultItem( + url=u, + fetch_status="error", + error=f"Web read service error: {str(e)}", + ) + for u in url_strings + ] + return view_models.FetchUrlResponse(results=results, total=len(results), succeeded=0, failed=len(results)) + + results = [] + for item in web_response.results: + if item.status != "success" or not item.content: + results.append( + view_models.FetchUrlResultItem( + url=item.url, + fetch_status="error", + error=item.error or "Failed to fetch or empty content", + ) + ) + continue + + # Build a safe filename from the page title or URL path + raw_name = item.title or urlparse(item.url).path.strip("/").replace("/", "_") or "page" + safe_name = re.sub(r"[^\w\s\-.]", "", raw_name).strip()[:200] or "page" + filename = f"{safe_name}.md" + + content_bytes = item.content.encode("utf-8") + content_size = len(content_bytes) + + # Wrap Markdown content as a virtual UploadFile (same interface as real file upload) + virtual_file = UploadFile( + filename=filename, + size=content_size, + headers=Headers({"content-type": "text/markdown"}), + file=io.BytesIO(content_bytes), + ) + + try: + upload_response = await self.upload_document(user_id, collection_id, virtual_file) + results.append( + view_models.FetchUrlResultItem( + url=item.url, + fetch_status="success", + document_id=upload_response.document_id, + filename=upload_response.filename, + size=upload_response.size, + status=str( + upload_response.status.value + if hasattr(upload_response.status, "value") + else upload_response.status + ), + ) + ) + except Exception as e: + logger.warning(f"Failed to upload fetched content for {item.url}: {e}") + results.append( + view_models.FetchUrlResultItem( + url=item.url, + fetch_status="error", + error=str(e), + ) + ) + + succeeded = sum(1 for r in results if r.fetch_status == "success") + return view_models.FetchUrlResponse( + results=results, + total=len(results), + succeeded=succeeded, + failed=len(results) - succeeded, + ) + # Create a global service instance for easy access # This uses the global db_ops instance and doesn't require session management in views diff --git a/aperag/service/graph_service.py b/aperag/service/graph_service.py index c1d9447ec..64de2a3ed 100644 --- a/aperag/service/graph_service.py +++ b/aperag/service/graph_service.py @@ -118,24 +118,36 @@ async def get_knowledge_graph( await rag.finalize_storages() def _convert_graph_to_dict(self, nodes, edges, is_truncated=False) -> Dict[str, Any]: - """Convert LightRAG graph objects to dictionary format""" + """ + Convert KnowledgeGraph to API dict. Semantics (see KnowledgeGraphNode): + - id: node identity and display key (storage must use entity_id). + - labels: pass-through from storage (e.g. [entity_type]); fallback to entity_id for display if empty. + - properties: entity_id, entity_type, description, source_id, file_path, entity_name. + """ def extract_properties(obj, default_fields): if hasattr(obj, "properties") and obj.properties: return obj.properties return {field: getattr(obj, field, None) for field in default_fields if hasattr(obj, field)} + default_node_fields = ["entity_id", "entity_name", "entity_type", "description", "source_id", "file_path"] + + def node_to_item(node): + props = extract_properties(node, default_node_fields) + # Use storage labels when present; else fallback so display is never numeric id + if getattr(node, "labels", None) and node.labels: + labels = node.labels + else: + display = props.get("entity_id") or props.get("entity_name") + labels = [display] if display is not None else ([node.id] if hasattr(node, "id") else []) + return { + "id": node.id, + "labels": labels, + "properties": props, + } + return { - "nodes": [ - { - "id": node.id, - "labels": [node.id] if hasattr(node, "id") else [], - "properties": extract_properties( - node, ["entity_id", "entity_type", "description", "source_id", "file_path"] - ), - } - for node in nodes - ], + "nodes": [node_to_item(node) for node in nodes], "edges": [ { "id": edge.id, @@ -428,6 +440,60 @@ async def _get_and_validate_collection(self, user_id: str, collection_id: str): return db_collection + async def search_entities( + self, + user_id: str, + collection_id: str, + query: str, + max_results: int = 50, + ) -> Dict[str, Any]: + """Search entities in the knowledge graph by name""" + db_collection = await self._get_and_validate_collection(user_id, collection_id) + + rag = await lightrag_manager.create_lightrag_instance(db_collection) + try: + kg: KnowledgeGraph = await rag.get_knowledge_graph( + node_label="*", + max_depth=1, + max_nodes=10000, + ) + query_lower = query.lower() + matching = [n for n in kg.nodes if query_lower in str(n.id).lower()] + result = self._convert_graph_to_dict(matching[:max_results], [], False) + return {"nodes": result["nodes"], "total": len(matching)} + finally: + await rag.finalize_storages() + + async def get_embedding_map( + self, + user_id: str, + collection_id: str, + max_nodes: int = 500, + ) -> Dict[str, Any]: + """Get entity list for embedding map visualization""" + db_collection = await self._get_and_validate_collection(user_id, collection_id) + + rag = await lightrag_manager.create_lightrag_instance(db_collection) + try: + kg: KnowledgeGraph = await rag.get_knowledge_graph( + node_label="*", + max_depth=1, + max_nodes=max_nodes * 2, + ) + # Compute per-node degree from edges + degree_map: Dict[str, int] = {} + for edge in kg.edges: + degree_map[edge.source] = degree_map.get(edge.source, 0) + 1 + degree_map[edge.target] = degree_map.get(edge.target, 0) + 1 + + nodes = kg.nodes[:max_nodes] + result = self._convert_graph_to_dict(nodes, [], False) + for node in result["nodes"]: + node["degree"] = degree_map.get(node["id"], 0) + return {"entities": result["nodes"], "total": len(nodes)} + finally: + await rag.finalize_storages() + async def export_for_kg_eval( self, user_id: str, collection_id: str, sample_size: int = 100000, include_source_texts: bool = True ) -> Dict[str, Any]: diff --git a/aperag/utils/audit_decorator.py b/aperag/utils/audit_decorator.py index 76296685d..df3bdc565 100644 --- a/aperag/utils/audit_decorator.py +++ b/aperag/utils/audit_decorator.py @@ -268,6 +268,18 @@ async def wrapper(*args, **kwargs): # Record end time end_time_ms = int(time.time() * 1000) + duration_ms = end_time_ms - start_time_ms + + # Emit a concise latency line for every audited endpoint so + # that the duration is visible in logs independently of whether + # the audit DB write succeeds. + logger.info( + "api %s %s (%s) status=200 duration_ms=%d", + request.method, + request.url.path, + actual_api_name, + duration_ms, + ) # Extract request data from function arguments (after parsing) request_data = _extract_request_data_from_args(request, kwargs) diff --git a/aperag/views/collections.py b/aperag/views/collections.py index f623c3c17..a8a178e60 100644 --- a/aperag/views/collections.py +++ b/aperag/views/collections.py @@ -259,6 +259,16 @@ async def list_documents_view( } +@router.get("/collections/{collection_id}/documents/staged", tags=["documents"]) +async def list_staged_documents_view( + request: Request, + collection_id: str, + user: User = Depends(required_user), +) -> view_models.StagedDocumentsResponse: + """Return all UPLOADED (staged) documents awaiting confirmation.""" + return await document_service.get_staged_documents(str(user.id), collection_id) + + @router.get("/collections/{collection_id}/documents/{document_id}", tags=["documents"]) async def get_document_view( request: Request, @@ -405,6 +415,25 @@ async def confirm_documents_view( return await document_service.confirm_documents(str(user.id), collection_id, confirm_request.document_ids) +@router.post("/collections/{collection_id}/documents/fetch-url", tags=["documents"]) +@audit(resource_type="document", api_name="FetchUrlDocument") +async def fetch_url_document_view( + request: Request, + collection_id: str, + fetch_request: view_models.FetchUrlRequest, + user: User = Depends(required_user), +) -> view_models.FetchUrlResponse: + """ + Fetch web page content from URLs and create UPLOADED documents. + + Each URL is fetched via the web read service (JINA with Trafilatura fallback). + Successful results are wrapped as virtual UploadFile objects and passed to + upload_document(), producing UPLOADED documents identical to file uploads. + Use POST /documents/confirm to move them to PENDING and start indexing. + """ + return await document_service.fetch_url_documents(str(user.id), collection_id, fetch_request.urls) + + @router.get("/collections/{collection_id}/graphs", tags=["graph"]) async def get_knowledge_graph_view( request: Request, @@ -430,3 +459,42 @@ async def get_knowledge_graph_view( raise HTTPException(status_code=404, detail="Collection not found") except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + + +@router.get("/collections/{collection_id}/graphs/embedding-map", tags=["graph"]) +async def get_graph_embedding_map_view( + request: Request, + collection_id: str, + max_nodes: int = Query(500, ge=1, le=5000), + user: User = Depends(required_user), +): + """Get entity list for embedding map visualization""" + from aperag.service.graph_service import graph_service + + try: + result = await graph_service.get_embedding_map(str(user.id), collection_id, max_nodes) + return result + except CollectionNotFoundException: + raise HTTPException(status_code=404, detail="Collection not found") + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.get("/collections/{collection_id}/graphs/entity-search", tags=["graph"]) +async def get_graph_entity_search_view( + request: Request, + collection_id: str, + q: str = Query(..., min_length=1, description="Search query"), + max_results: int = Query(50, ge=1, le=500), + user: User = Depends(required_user), +): + """Search entities in the knowledge graph by name""" + from aperag.service.graph_service import graph_service + + try: + result = await graph_service.search_entities(str(user.id), collection_id, q, max_results) + return result + except CollectionNotFoundException: + raise HTTPException(status_code=404, detail="Collection not found") + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) diff --git a/docs/en-US/integration/dify.md b/docs/en-US/integration/dify.md index f05e7d201..4d45f656e 100644 --- a/docs/en-US/integration/dify.md +++ b/docs/en-US/integration/dify.md @@ -31,7 +31,7 @@ ApeRAG is a production-grade RAG platform with multimodal indexing, AI agents, M ## Step 1: Prepare Knowledge Base -Visit ApeRAG at https://rag.apecloud.com/ , register/login, and select or import a knowledge base. Here we use the Romance of the Three Kingdoms example - click subscribe. +Open your ApeRAG web UI (see [Quick Start](../../../README.md#quick-start); with Docker Compose this is typically http://localhost:3000/web/). Sign in and select or import a knowledge base. This walkthrough uses the Romance of the Three Kingdoms example—click **Subscribe**.
Subscribe to Collection @@ -49,7 +49,7 @@ Go to Dify - Tools - MCP, click Add MCP Server. ### 2.2 Fill Configuration -Fill in Server URL: `https://rag.apecloud.com/mcp/` and your API Key copied from ApeRAG, then click Confirm. +Fill in Server URL: `http://localhost:8000/mcp/` (use `https:///mcp/` if ApeRAG is not local), paste your API Key from ApeRAG, then click Confirm.
Configure MCP diff --git a/docs/en-US/integration/mcp-api.md b/docs/en-US/integration/mcp-api.md index 2c1275b17..85af530ef 100644 --- a/docs/en-US/integration/mcp-api.md +++ b/docs/en-US/integration/mcp-api.md @@ -17,7 +17,7 @@ For Claude Desktop, add to configuration file: { "mcpServers": { "aperag": { - "url": "https://rag.apecloud.com/mcp/", + "url": "http://localhost:8000/mcp/", "headers": { "Authorization": "Bearer your-api-key-here" } diff --git a/docs/zh-CN/design/search_flow_design.md b/docs/zh-CN/design/search_flow_design.md new file mode 100644 index 000000000..29c399b67 --- /dev/null +++ b/docs/zh-CN/design/search_flow_design.md @@ -0,0 +1,654 @@ +--- +title: 检索流程设计 +description: ApeRAG 检索流程的完整设计文档，涵盖 MCP 入口、Flow 执行引擎与各检索类型的核心实现 +keywords: 检索, Flow Engine, DAG, 向量检索, 全文检索, 图检索, MCP +position: 3 +--- + +# ApeRAG 检索流程设计 + +## 概述 + +ApeRAG 的检索流程采用 **Flow 执行引擎**驱动的多路并行检索架构。用户（或 AI Agent）通过 MCP 工具发起检索请求，请求到达服务端后被转化为一个有向无环图（DAG）描述的检索 Flow，由 Flow 引擎按拓扑顺序并行执行各检索节点，最终将多路结果合并重排后返回。 + +```mermaid +graph LR + A[AI Agent / 用户] -->|MCP 工具调用| B[MCP Server] + B -->|"POST /api/v1/collections//searches"| C[FastAPI 路由] + C --> D[CollectionService.create_search] + D --> E[execute_search_flow\n动态构建 DAG] + E --> F[FlowEngine.execute_flow\nDAG 拓扑排序 + 并行执行] + + F --> G1[vector_search\n向量检索] + F --> G2[fulltext_search\n全文检索] + F --> G3[graph_search\n图谱检索] + F --> G4[summary_search\n摘要检索] + F --> G5[vision_search\n视觉检索] + + G1 --> H[merge\n多路结果合并] + G2 --> H + G3 --> H + G4 --> H + G5 --> H + + H --> I[rerank\n结果重排] + I --> J[SearchResult 返回给调用方] +``` + +图中 REST 路径里的 `` 表示路径参数，与 OpenAPI 写法 `/collections/{collection_id}/searches` 含义相同（Mermaid 中花括号 `{}` 为语法保留字符，故图中用尖括号表示占位符）。 + +--- + +## 第一层：MCP 入口 + +[MCP（Model Context Protocol）](https://modelcontextprotocol.io/) 是 ApeRAG 面向 AI Agent 暴露能力的标准接口。Agent 无需直接调用 REST API，只需调用 MCP 工具即可完成检索。 + +### MCP 挂载位置 + +MCP Server 以 Stateless HTTP 模式挂载在 FastAPI 应用的 `/mcp` 路径下： + +```python +# aperag/app.py +mcp_app = mcp_server.http_app(path="/", stateless_http=True) +app.mount("/mcp", mcp_app) +``` + +### `search_collection` 工具 + +最核心的检索工具是 `search_collection`，它封装了对 REST API 的调用： + +```python +# aperag/mcp/server.py +@mcp_server.tool +async def search_collection( + collection_id: str, + query: str, + use_vector_index: bool = True, + use_fulltext_index: bool = True, + use_graph_index: bool = True, + use_summary_index: bool = True, + use_vision_index: bool = True, + rerank: bool = True, + topk: int = 5, + query_keywords: list[str] = None, +) -> Dict[str, Any]: + """Search for knowledge in a persistent collection/knowledge base""" + ... + async with httpx.AsyncClient(timeout=120.0) as client: + response = await client.post( + f"{API_BASE_URL}/api/v1/collections/{collection_id}/searches", + headers={"Authorization": f"Bearer {api_key}"}, + json=search_data, + ) +``` + +工具会将参数（启用哪些检索类型、topk、关键词等）组装成 JSON，以 Bearer Token 的方式调用内部 REST API。 + +> **注意**：`API_BASE_URL` 默认为 `http://localhost:8000`，与 API 进程同机部署时合理。若 MCP 与 API 分离部署，需要通过环境变量或配置覆盖此地址。 + +--- + +## 第二层：REST API 端点 + +MCP 工具调用最终落到以下 FastAPI 路由： + +```python +# aperag/views/collections.py +@router.post("/collections/{collection_id}/searches", tags=["search"]) +@audit(resource_type="search", api_name="CreateSearch") +async def create_search_view( + request: Request, + collection_id: str, + data: view_models.SearchRequest, + user: User = Depends(required_user), +) -> view_models.SearchResult: + return await collection_service.create_search(str(user.id), collection_id, data) +``` + +### SearchRequest 结构 + +请求体 `SearchRequest` 对应 OpenAPI schema，字段如下： + +| 字段 | 类型 | 说明 | +|------|------|------| +| `query` | string | 用户查询语句 | +| `vector_search` | object | 向量检索参数（`topk`、`similarity`） | +| `fulltext_search` | object | 全文检索参数（`topk`、`keywords`） | +| `graph_search` | object | 图谱检索参数（`topk`） | +| `summary_search` | object | 摘要检索参数（`topk`、`similarity`） | +| `vision_search` | object | 视觉检索参数（`topk`、`similarity`） | +| `rerank` | boolean | 是否对结果重排 | +| `save_to_history` | boolean | 是否保存到搜索历史 | + +某个检索类型的字段为 `null`/缺失时，表示本次检索**不启用**该类型。 + +--- + +## 第三层：`execute_search_flow` —— 动态构建检索 DAG + +`create_search` 校验权限后，调用 `execute_search_flow` 动态构建并执行检索 Flow： + +```python +# aperag/service/collection_service.py +async def execute_search_flow( + self, + data: view_models.SearchRequest, + collection_id: str, + search_user_id: str, + chat_id: Optional[str] = None, + flow_name: str = "search", + flow_title: str = "Search", +) -> Tuple[List[SearchResultItem], str]: +``` + +### 构建过程 + +这个方法根据 `SearchRequest` 的内容，**动态**决定要创建哪些节点，再把它们连接成 DAG： + +```python +nodes = {} +edges = [] +merge_node_id = "merge" + +# 按需添加各检索节点 +if data.vector_search: + nodes["vector_search"] = NodeInstance(type="vector_search", ...) + edges.append(Edge(source="vector_search", target="merge")) + +if data.fulltext_search: + nodes["fulltext_search"] = NodeInstance(type="fulltext_search", ...) + edges.append(Edge(source="fulltext_search", target="merge")) + +if data.graph_search: + nodes["graph_search"] = NodeInstance(type="graph_search", ...) + edges.append(Edge(source="graph_search", target="merge")) + +# ... summary_search, vision_search 同理 + +# merge 节点始终存在 +nodes["merge"] = NodeInstance(type="merge", ...) + +# rerank 节点始终在 merge 之后 +nodes["rerank"] = NodeInstance(type="rerank", ...) +edges.append(Edge(source="merge", target="rerank")) +``` + +构建完成后，将 `FlowInstance` 交给 `FlowEngine` 执行： + +```python +flow = FlowInstance(name=flow_name, nodes=nodes, edges=edges) +engine = FlowEngine() +result, _ = await engine.execute_flow(flow, initial_data={"query": query, "user": search_user_id}) +``` + +### 典型 DAG 示意（全部检索类型启用） + +``` +vector_search ──┐ +fulltext_search ─┤ +graph_search ────┼──→ merge ──→ rerank ──→ 返回结果 +summary_search ──┤ +vision_search ───┘ +``` + +检索节点之间**没有依赖关系**，全部指向 `merge`；`merge` 完成后，结果流向 `rerank`。 + +--- + +## 第四层：Flow 执行引擎 + +Flow 引擎（`aperag/flow/engine.py`）是整个检索流程的核心调度组件，负责解析 DAG、拓扑排序、并行执行。 + +### 核心数据模型 + +```python +# aperag/flow/base/models.py + +class FlowInstance(BaseModel): + name: str + title: str + nodes: Dict[str, NodeInstance] # node_id -> NodeInstance + edges: List[Edge] # 有向边列表 + +class NodeInstance(BaseModel): + id: str + type: str # 节点类型，对应已注册的 Runner + input_values: dict # 节点输入参数（支持 Jinja 模板引用其他节点输出） + +class Edge(BaseModel): + source: str # 源节点 id + target: str # 目标节点 id（target 依赖 source） +``` + +### 执行流程 + +`FlowEngine.execute_flow` 的完整执行步骤： + +```mermaid +flowchart TB + A[execute_flow 入口] --> B[写入 initial_data 到 ExecutionContext\nquery / user / chat_id 等] + B --> C[_topological_sort\nKahn 算法拓扑排序] + C --> D{是否有环?} + D -- 有环 --> E[抛出 CycleError] + D -- 无环 --> F[_find_parallel_groups\n按层分组] + F --> G[逐层执行 _execute_node_group] + G --> H{当前层节点数} + H -- 1个 --> I[顺序执行单个节点] + H -- 多个 --> J[asyncio.gather 并行执行] + I --> K[写入节点输出到 context.outputs] + J --> K + K --> L{还有下一层?} + L -- 是 --> G + L -- 否 --> M[返回 context.outputs] +``` + +### 拓扑排序：Kahn 算法 + +```python +def _topological_sort(self, flow: FlowInstance) -> List[str]: + # 统计每个节点的入度（有多少条边指向它） + in_degree = {node_id: 0 for node_id in flow.nodes} + for edge in flow.edges: + in_degree[edge.target] += 1 + + # 从入度为 0 的节点开始（无依赖的节点） + queue = deque([node_id for node_id, degree in in_degree.items() if degree == 0]) + + sorted_nodes = [] + while queue: + node_id = queue.popleft() + sorted_nodes.append(node_id) + # 处理完当前节点后，更新后继节点的入度 + for edge in flow.edges: + if edge.source == node_id: + in_degree[edge.target] -= 1 + if in_degree[edge.target] == 0: + queue.append(edge.target) + + # 若处理节点数不足，说明图中存在环 + if len(sorted_nodes) != len(flow.nodes): + raise CycleError("Flow contains cycles") + + return sorted_nodes +``` + +### 并行分层执行 + +拓扑排序后，引擎进一步将节点分成**可以并行的层（Level）**： + +```python +def _find_parallel_groups(self, flow, sorted_nodes) -> List[Set[str]]: + """将拓扑序的节点按可并行的层分组""" + in_degree = {node_id: 0 for node_id in flow.nodes} + for edge in flow.edges: + in_degree[edge.target] += 1 + + processed = set() + groups = [] + + while len(processed) < len(sorted_nodes): + # 找出当前所有入度为 0 且未处理的节点 → 可以并行 + current_group = { + node_id for node_id in sorted_nodes + if in_degree[node_id] == 0 and node_id not in processed + } + groups.append(current_group) + for node_id in current_group: + processed.add(node_id) + for edge in flow.edges: + if edge.source == node_id: + in_degree[edge.target] -= 1 + + return groups # 每个元素是一个 Set，同组内可并行 +``` + +对于默认检索 Flow，分层结果如下： + +| 层 | 节点（可并行） | +|----|--------------| +| 第 1 层 | `vector_search`、`fulltext_search`、`graph_search`、`summary_search`、`vision_search` | +| 第 2 层 | `merge`（等待第 1 层全部完成） | +| 第 3 层 | `rerank`（等待 merge 完成） | + +同一层内的节点通过 `asyncio.gather` **并发执行**，显著降低多路检索的总延迟。 + +### 节点间数据传递：Jinja 模板 + +节点的 `input_values` 支持 Jinja2 模板语法，用于引用其他节点的输出： + +```python +# merge 节点引用各检索节点的输出 +merge_node_values = { + "vector_search_docs": "{{ nodes.vector_search.output.docs }}", + "fulltext_search_docs": "{{ nodes.fulltext_search.output.docs }}", + "graph_search_docs": "{{ nodes.graph_search.output.docs }}", + ... +} + +# rerank 节点引用 merge 节点的输出 +rerank_input_values = { + "docs": "{{ nodes.merge.output.docs }}", + ... +} +``` + +引擎在执行节点前会先解析这些模板，将前序节点的实际输出填充进来。 + +### NodeRunner 注册机制 + +每种节点类型通过装饰器注册到全局注册表 `NODE_RUNNER_REGISTRY`： + +```python +# aperag/flow/base/models.py +NODE_RUNNER_REGISTRY = {} + +def register_node_runner(node_type, input_model, output_model): + def decorator(cls): + NODE_RUNNER_REGISTRY[node_type] = { + "runner": cls(), + "input_model": input_model, + "output_model": output_model, + } + return cls + return decorator +``` + +节点 Runner 示例： + +```python +# aperag/flow/runners/vector_search.py +@register_node_runner( + "vector_search", + input_model=VectorSearchInput, + output_model=VectorSearchOutput, +) +class VectorSearchNodeRunner(BaseNodeRunner): + async def run(self, ui: VectorSearchInput, si: SystemInput) -> Tuple[VectorSearchOutput, dict]: + ... +``` + +`import aperag.flow.runners` 时，所有 Runner 模块被加载，完成注册（见 `engine.py` 第 23 行的 `import` 语句）。 + +--- + +## 第五层：各检索类型详解 + +### 1. 向量检索（`vector_search`） + +**原理**：将用户查询通过 Embedding 模型转为向量，在向量数据库中做近似最近邻搜索，找出语义最相似的文档片段。 + +**适用场景**：语义理解类查询，例如"有没有关于性能优化的内容"。 + +**核心代码**（`aperag/flow/runners/vector_search.py`）： + +```python +# 1. 生成查询向量 +vector = embedding_model.embed_query(query) + +# 2. 在向量数据库中查询 +results = context_manager.query( + query, + score_threshold=similarity_threshold, + topk=top_k, + vector=vector, + index_types=["vector"], + chat_id=chat_id, +) + +# 3. 标记召回类型 +for item in results: + item.metadata["recall_type"] = "vector_search" +``` + +**输入参数**： + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `top_k` | 返回结果数量上限 | 5 | +| `similarity_threshold` | 相似度阈值，低于此值的结果过滤掉 | 0.2 | +| `collection_ids` | 检索的知识库 ID 列表 | — | +| `chat_id` | 会话 ID（会话文件检索时用于过滤） | null | + +--- + +### 2. 全文检索（`fulltext_search`） + +**原理**：基于关键词的倒排索引检索，支持精确词匹配和布尔查询。 + +**适用场景**：精确词语查询，例如"找出包含'PostgreSQL'的段落"。 + +**核心代码**（`aperag/flow/runners/fulltext_search.py`）： + +```python +# 支持自定义关键词，或从查询中自动提取 +if not keywords: + keywords = extract_keywords(query) + +results = await fulltext_indexer.search( + index=index_name, + query=query, + keywords=keywords, + topk=top_k, + chat_id=chat_id, +) +``` + +**输入参数**： + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `top_k` | 返回结果数量上限 | 5 | +| `keywords` | 自定义关键词列表（可选，不传则从 query 自动提取） | [] | +| `collection_ids` | 检索的知识库 ID 列表 | — | +| `chat_id` | 会话 ID | null | + +--- + +### 3. 图谱检索（`graph_search`） + +**原理**：基于知识图谱（Knowledge Graph）的检索。文档在建索引时，LightRAG 会从中提取实体和关系，构建图谱。检索时，在图谱中做 hybrid 模式（向量 + 关键词）的子图查询，返回相关的实体上下文。 + +**适用场景**：需要多跳推理的查询，例如"张三负责的团队用到了哪些技术栈"。 + +**前提条件**：知识库必须启用 `enable_knowledge_graph` 选项。 + +**核心代码**（`aperag/flow/runners/graph_search.py`）： + +```python +# 需要集合开启了知识图谱 +if not config.enable_knowledge_graph: + return [] + +# 创建 LightRAG 实例并查询 +rag = await lightrag_manager.create_lightrag_instance(collection) +param = QueryParam( + mode="hybrid", # 向量 + 关键词混合查询 + only_need_context=True, # 只需要图谱上下文，不需要 LLM 生成 + top_k=top_k, +) +context = await rag.aquery_context(query, param=param) +``` + +**输入参数**： + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `top_k` | 返回结果数量上限 | 5 | +| `collection_ids` | 检索的知识库 ID 列表 | — | + +--- + +### 4. 摘要检索（`summary_search`） + +**原理**：每个文档在建索引时会生成文档级别的摘要向量。检索时对摘要向量做近似最近邻搜索，以文档整体粒度召回，适合"找和这个主题相关的文档"类查询。 + +**适用场景**：需要文档级别召回（而非段落级别）的场景。 + +**核心代码**（`aperag/flow/runners/summary_search.py`）： + +```python +results = context_manager.query( + query, + score_threshold=similarity_threshold, + topk=top_k, + vector=vector, + index_types=["summary"], # 只查摘要索引 + chat_id=chat_id, +) +for item in results: + item.metadata["recall_type"] = "summary_search" +``` + +**输入参数**： + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `top_k` | 返回结果数量上限 | 5 | +| `similarity` | 相似度阈值 | 0.2 | +| `collection_ids` | 检索的知识库 ID 列表 | — | + +--- + +### 5. 视觉检索（`vision_search`） + +**原理**：基于多模态 Embedding 对图像内容做向量检索。文档中的图片在建索引时会生成视觉向量，支持用自然语言描述来检索相关图片。 + +**适用场景**：包含大量图表、截图的知识库，例如"找一张关于系统架构的图"。 + +**输入参数**： + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `top_k` | 返回结果数量上限 | 5 | +| `similarity` | 相似度阈值 | 0.2 | +| `collection_ids` | 检索的知识库 ID 列表 | — | + +--- + +### 6. 合并节点（`merge`） + +所有检索节点的结果汇聚到 `merge` 节点： + +```python +# aperag/flow/runners/merge.py +@register_node_runner("merge", input_model=MergeInput, output_model=MergeOutput) +class MergeNodeRunner(BaseNodeRunner): + async def run(self, ui: MergeInput, si: SystemInput): + # 合并所有检索路的 docs + all_docs = ( + ui.vector_search_docs + + ui.fulltext_search_docs + + ui.graph_search_docs + + ui.summary_search_docs + + ui.vision_search_docs + ) + + # 按文本内容去重 + if ui.deduplicate: + seen = set() + unique_docs = [] + for doc in all_docs: + if doc.text not in seen: + seen.add(doc.text) + unique_docs.append(doc) + return MergeOutput(docs=unique_docs), {} + + return MergeOutput(docs=all_docs), {} +``` + +Merge 策略： +- **合并策略（`merge_strategy`）**：目前为 `union`（取并集） +- **去重（`deduplicate`）**：默认开启，按文本内容去重，避免同一段落被多个检索路召回 + +--- + +### 7. 重排节点（`rerank`） + +Rerank 节点对合并后的文档列表按与查询的相关性重新排序，过滤噪声，提升最终结果质量： + +```python +# aperag/flow/runners/rerank.py +@register_node_runner("rerank", input_model=RerankInput, output_model=RerankOutput) +class RerankNodeRunner(BaseNodeRunner): + async def run(self, ui: RerankInput, si: SystemInput): + if ui.use_rerank_service: + # 调用专用 Rerank 模型服务（如 Jina Reranker、Cohere 等） + rerank_service = RerankService(...) + docs = await rerank_service.rerank(si.query, ui.docs) + else: + # 降级策略：按原始召回分数排序 + docs = sorted(ui.docs, key=lambda d: d.score, reverse=True) + return RerankOutput(docs=docs), {} +``` + +Rerank 行为由 `SearchRequest.rerank` 字段控制： +- `rerank=true`：尝试调用用户配置的 Rerank 模型服务；若未配置，降级为按分数排序 +- `rerank=false`：直接按合并后的召回分数排序 + +--- + +## 检索结果 + +流程执行完成后，`rerank` 节点的输出 `docs` 被转换为 `SearchResultItem` 列表： + +```python +for idx, doc in enumerate(docs): + items.append(SearchResultItem( + rank=idx + 1, + score=doc.score, + content=doc.text, + source=doc.metadata.get("source", ""), + recall_type=doc.metadata.get("recall_type", ""), # 标明来自哪种检索 + metadata=doc.metadata, + )) +``` + +`recall_type` 字段枚举值： + +| 值 | 含义 | +|----|------| +| `vector_search` | 来自向量检索 | +| `fulltext_search` | 来自全文检索 | +| `graph_search` | 来自图谱检索 | +| `summary_search` | 来自摘要检索 | +| `vision_search` | 来自视觉检索 | + +--- + +## 会话文件检索 + +`execute_search_flow` 也被复用于会话（Chat）内的临时文件检索，入口为： + +``` +POST /api/v1/chats/{chat_id}/search +``` + +区别在于传入了 `chat_id` 参数，各检索节点会利用该参数过滤，只检索属于该会话的上传文件，不会混入知识库的全局文档。 + +--- + +## 关键文件索引 + +| 文件 | 职责 | +|------|------| +| `aperag/mcp/server.py` | MCP 工具定义，`search_collection` 入口 | +| `aperag/app.py` | FastAPI 应用，MCP 挂载点 | +| `aperag/views/collections.py` | REST API 路由 `/collections/{id}/searches` | +| `aperag/service/collection_service.py` | `create_search` 和 `execute_search_flow` 实现 | +| `aperag/flow/engine.py` | Flow 执行引擎，拓扑排序与并行调度 | +| `aperag/flow/base/models.py` | DAG 数据模型，NodeRunner 注册机制 | +| `aperag/flow/runners/vector_search.py` | 向量检索 Runner | +| `aperag/flow/runners/fulltext_search.py` | 全文检索 Runner | +| `aperag/flow/runners/graph_search.py` | 图谱检索 Runner（基于 LightRAG） | +| `aperag/flow/runners/summary_search.py` | 摘要检索 Runner | +| `aperag/flow/runners/vision_search.py` | 视觉检索 Runner | +| `aperag/flow/runners/merge.py` | 多路结果合并 Runner | +| `aperag/flow/runners/rerank.py` | 结果重排 Runner | + +--- + +## 相关文档 + +- [索引链路架构设计](./indexing_architecture.md)：了解各检索类型的索引是如何构建的 +- [图索引构建流程](./graph_index_creation.md)：深入了解知识图谱的构建过程 +- [MCP API 集成指南](../integration/mcp-api.md)：如何在 Agent 中接入 ApeRAG 的 MCP 工具 diff --git a/docs/zh-CN/design/url_and_text_import_design.md b/docs/zh-CN/design/url_and_text_import_design.md new file mode 100644 index 000000000..75955c015 --- /dev/null +++ b/docs/zh-CN/design/url_and_text_import_design.md @@ -0,0 +1,590 @@ +--- +title: URL 与文本导入设计 +position: 4 +--- + +# Collection 文档导入扩展：URL 抓取与文本粘贴 + +## 概述 + +本文档描述在 ApeRAG Collection 中新增两种文档来源方式的设计： + +1. **URL 导入**：用户输入网址，系统自动调用 `web/read` 接口抓取页面内容，生成 Markdown 文件，走现有两阶段上传流程入库。 +2. **文本导入**：用户在前端粘贴文本，前端直接将其封装为 `.txt` 文件，调用现有上传接口，**完全无需新增后端代码**。 + +两种方式都只是"给现有上传流程提供文件内容的方式"，confirm 及后续索引构建完全复用现有逻辑。 + +> **范围说明**：本期不包含"根据文字搜索网络并导入"功能，但架构设计保留此扩展空间。 + +--- + +## 设计原则 + +> URL 抓取和文本粘贴只是"选择文件"的替代方式。 + +一旦内容到手（Markdown 字符串 / 文本字符串），它就被包装成一个虚拟文件，走与普通文件上传完全相同的路径： + +``` +[来源] [统一入口] [后续流程（不变）] +文件选择 ──────────► upload_document() ──► UPLOADED ──► confirm ──► 索引构建 +URL 抓取 ──────────►（虚拟 UploadFile） +文本粘贴 ──────────►（前端 File 对象） +``` + +--- + +## 现状与可复用组件 + +### 现有两阶段上传流程 + +``` +Step 1: POST /collections/{id}/documents/upload → status = UPLOADED（临时） +Step 2: POST /collections/{id}/documents/confirm → status = PENDING → 触发索引构建 +``` + +URL 导入和文本导入都将产出 `UPLOADED` 状态的文档，与文件上传无缝衔接。 + +### 关键可复用组件 + +| 组件 | 位置 | 如何复用 | +|------|------|---------| +| 文档上传服务 | `aperag/service/document_service.py` → `upload_document()` | URL 导入后端调用此方法存储抓取内容 | +| 文档确认服务 | `document_service.confirm_documents()` | 完全不变 | +| Web Read 接口 | `POST /api/v1/web/read`（`aperag/views/web.py`） | URL 导入后端通过 HTTP 调用此接口 | +| ReaderService | `aperag/websearch/reader/reader_service.py` | web/read 的底层实现（JINA + Trafilatura fallback） | +| 文档列表页暂存区 | `document-upload.tsx` | URL/文本产出的 `UPLOADED` 文档自动出现在此列表 | + +--- + +## 架构设计 + +### 总体流程 + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Frontend (Next.js) │ +│ │ +│ ┌──────────────┐ ┌─────────────────┐ ┌──────────────────┐ │ +│ │ 📁 选择文件 │ │ 🔗 输入网址 │ │ 📋 粘贴文字 │ │ +│ │ (现有) │ │ (新增) │ │ (新增) │ │ +│ └──────┬───────┘ └────────┬────────┘ └────────┬─────────┘ │ +│ │ │ │ │ +│ │ POST /fetch-url │ │ +│ │ │ new File([text], "x.txt") │ +│ │ │ │ │ +│ └───────────────────┴────────────────────┘ │ +│ │ │ +│ POST /documents/upload │ +│ （现有接口，所有来源统一入口） │ +└─────────────────────────────┬───────────────────────────────────┘ + │ + ▼ + ┌───────────────────────────────┐ + │ document_service │ + │ .upload_document() │ + │ status = UPLOADED │ + └───────────────┬───────────────┘ + │ + 用户在暂存区点击"保存到知识库" + │ + POST /documents/confirm + │ + ▼ + ┌───────────────────────────────┐ + │ document_service │ + │ .confirm_documents() │ + │ status: UPLOADED → PENDING │ + │ 创建 DocumentIndex 记录 │ + │ 触发 reconcile 任务 │ + └───────────────────────────────┘ +``` + +--- + +## URL 导入详细设计 + +### 新增后端接口 + +**接口**：`POST /api/v1/collections/{collection_id}/documents/fetch-url` + +此接口是本次需求唯一新增的后端接口。 + +**职责**： +1. 接收用户提交的 URL 列表 +2. 通过 HTTP 调用内部 `POST /api/v1/web/read` 接口抓取页面内容 +3. 将每个 URL 的抓取结果包装为虚拟 `UploadFile` 对象 +4. 调用现有 `document_service.upload_document()` 存储到对象存储（status=UPLOADED） +5. 返回创建的 `document_id` 列表，前端将其合并到暂存区 + +**Request Body**： + +```json +{ + "urls": [ + "https://example.com/article1", + "https://example.com/article2" + ] +} +``` + +| 字段 | 类型 | 必填 | 约束 | +|------|------|------|------| +| `urls` | `string[]` | ✅ | 1~10 个，必须是合法 http/https URL | + +**Response**（200 OK）： + +```json +{ + "documents": [ + { + "id": "doc_abc123", + "name": "示例文章标题.md", + "status": "UPLOADED", + "size": 8192, + "url": "https://example.com/article1", + "fetch_status": "success" + }, + { + "id": null, + "name": null, + "status": null, + "url": "https://example.com/article2", + "fetch_status": "error", + "error": "页面无法访问（403）" + } + ], + "total": 2, + "succeeded": 1, + "failed": 1 +} +``` + +**说明**： +- 接口同步执行（URL 数量限制在 10 个以内，抓取过程在接口请求内完成） +- 部分 URL 失败不影响其他 URL，前端针对失败项目展示错误信息 +- 成功的文档处于 `UPLOADED` 状态，出现在前端暂存区供用户确认 + +### 后端实现逻辑 + +在 `aperag/views/collections.py` 中新增路由函数（约 60 行）： + +```python +@router.post("/collections/{collection_id}/documents/fetch-url", tags=["documents"]) +@audit(resource_type="document", api_name="FetchUrlDocument") +async def fetch_url_document_view( + request: Request, + collection_id: str, + body: view_models.FetchUrlRequest, + user: User = Depends(required_user), +) -> view_models.FetchUrlResponse: + """ + Fetch web page content from URLs and create UPLOADED documents. + + Internally calls POST /api/v1/web/read to retrieve page content, + then wraps each result as a virtual UploadFile and calls + document_service.upload_document() to persist as UPLOADED documents. + """ + results = [] + + # Step 1: Call web/read service layer (via HTTP to /api/v1/web/read) + web_read_request = WebReadRequest(url_list=body.urls, timeout=30) + web_read_response = await _call_web_read(web_read_request, user) + + # Step 2: For each result, wrap as UploadFile and call upload_document() + for item in web_read_response.results: + if item.status != "success" or not item.content: + results.append(FetchUrlResultItem( + url=item.url, + fetch_status="error", + error=item.error or "Failed to fetch content", + )) + continue + + # Determine filename from page title or URL + filename = _url_to_filename(item.title, item.url) + + # Wrap Markdown content as a virtual UploadFile + virtual_file = _make_upload_file(filename, item.content.encode("utf-8")) + + try: + doc = await document_service.upload_document( + user=str(user.id), + collection_id=collection_id, + file=virtual_file, + extra_metadata={"source_url": item.url, "source_type": "url"}, + ) + results.append(FetchUrlResultItem( + url=item.url, + fetch_status="success", + document=doc, + )) + except Exception as e: + results.append(FetchUrlResultItem( + url=item.url, + fetch_status="error", + error=str(e), + )) + + return FetchUrlResponse( + documents=results, + total=len(results), + succeeded=sum(1 for r in results if r.fetch_status == "success"), + failed=sum(1 for r in results if r.fetch_status == "error"), + ) +``` + +### 调用 web/read 的方式 + +采用**调用 Service 层**而非发起内部 HTTP 请求，直接复用 `web.py` 中的 `_read_with_jina_fallback` / `_read_with_trafilatura_only` 私有函数（或将其提取为 `reader_service` 的共享方法）： + +```python +async def _call_web_read(request: WebReadRequest, user: User) -> WebReadResponse: + """Call web read service layer, with JINA + Trafilatura fallback.""" + jina_api_key = await _get_user_jina_api_key(user) + if jina_api_key: + return await _read_with_jina_fallback(request, jina_api_key) + else: + return await _read_with_trafilatura_only(request) +``` + +> 这些私有函数已在 `aperag/views/web.py` 中实现，将其提取到 `aperag/websearch/reader/reader_service.py` 的公共方法即可被两处复用，保持模块化边界。 + +--- + +## 文本导入详细设计 + +### 零后端改动 + +文本导入**不需要新增任何后端接口**。前端在客户端将用户粘贴的文本封装为标准的 `File` 对象，调用现有上传接口： + +```typescript +// web/src/app/.../import/text-import.tsx + +const handleImport = async () => { + const filename = title.trim() ? `${title.trim()}.txt` : `note-${Date.now()}.txt`; + const file = new File([textContent], filename, { type: "text/plain" }); + + // Reuse existing upload API — no new backend endpoint needed + const response = await apiClient.defaultApi.collectionsCollectionIdDocumentsUploadPost({ + collectionId: collection.id, + file, + }); + + // Add to staging area (same as file upload) + onDocumentUploaded(response.data); +}; +``` + +这样文本文档与文件上传产出完全相同的结果（`UPLOADED` 状态的 Document），在暂存区中一视同仁。 + +--- + +## 前端设计 + +### 入口 Dialog + +在文档列表页的"添加文档"按钮点击后，显示来源选择 Dialog： + +``` +┌──────────────────────────────────────────────────┐ +│ 向知识库中添加文档 [×] │ +├──────────────────────────────────────────────────┤ +│ │ +│ ┌────────────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ 📁 │ │ 🔗 │ │ 📋 │ │ +│ │ 上传文件 │ │ 网址 │ │ 粘贴文字 │ │ +│ └────────────────┘ └──────────┘ └──────────┘ │ +│ │ +└──────────────────────────────────────────────────┘ +``` + +### 网址导入表单（`url-import.tsx`） + +``` +┌──────────────────────────────────────────────────┐ +│ ← 网址导入 [×] │ +├──────────────────────────────────────────────────┤ +│ 粘贴网址，系统将自动抓取页面内容导入知识库。 │ +│ │ +│ ┌──────────────────────────────────────────┐ │ +│ │ https://example.com/article1 │ │ +│ │ https://example.com/article2 │ │ +│ │ │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ • 每行输入一个网址，最多 10 个 │ +│ • 仅支持公开可访问的网页 │ +│ • 需要登录的页面无法抓取 │ +│ │ +│ [取消] [抓取并添加] │ +└──────────────────────────────────────────────────┘ +``` + +点击"抓取并添加"后： +1. 调用 `POST /collections/{id}/documents/fetch-url` +2. 成功的 URL 对应文档出现在上传暂存区（同文件上传） +3. 失败的 URL 在 Dialog 内以红色错误信息展示 +4. Dialog 关闭，用户在暂存区一起 confirm + +### 粘贴文字表单（`text-import.tsx`） + +``` +┌──────────────────────────────────────────────────┐ +│ ← 粘贴文字 [×] │ +├──────────────────────────────────────────────────┤ +│ 粘贴文字内容，即可将其导入知识库。 │ +│ │ +│ 标题（可选） │ +│ ┌──────────────────────────────────────────┐ │ +│ │ 我的笔记 │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ 内容 │ +│ ┌──────────────────────────────────────────┐ │ +│ │ 在此处粘贴文字… │ │ +│ │ │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ [取消] [添加] │ +└──────────────────────────────────────────────────┘ +``` + +点击"添加"后： +1. 前端创建 `new File([content], "${title}.txt")` 对象 +2. 调用现有 `POST /documents/upload` 接口（无感知） +3. 文档出现在上传暂存区 +4. Dialog 关闭，用户在暂存区 confirm + +### 前端暂存区（扩展现有 `document-upload.tsx`） + +现有暂存区已支持展示所有 `UPLOADED` 状态文档。URL/文本产出的文档与文件上传文档合并展示，confirm 操作完全不变： + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 暂存区（待确认） │ +├───────────────────────────────────────┬──────┬─────────────┤ +│ 文档名 │ 大小 │ 状态 │ +├───────────────────────────────────────┼──────┼─────────────┤ +│ 📄 user_manual.pdf │ 5 MB │ ✅ 已上传 │ +│ 🌐 示例文章标题.md（来自 URL） │ 8 KB │ ✅ 已上传 │ +│ 📝 我的笔记.txt │ 2 KB │ ✅ 已上传 │ +│ 🌐 example.com/article2（来自 URL） │ — │ ❌ 抓取失败 │ +└───────────────────────────────────────┴──────┴─────────────┘ + [保存到知识库（3 个文档）] +``` + +### 前端组件结构 + +``` +web/src/app/workspace/collections/[collectionId]/documents/ +├── page.tsx # 文档列表页（现有，加"添加"按钮入口） +└── upload/ + ├── page.tsx # 上传页（现有） + ├── document-upload.tsx # 暂存区组件（现有，几乎不改） + └── import/ + ├── import-dialog.tsx # 来源选择 Dialog（新增） + ├── url-import.tsx # URL 输入表单（新增） + └── text-import.tsx # 文本粘贴表单（新增） +``` + +--- + +## 新增代码量统计 + +| 位置 | 改动类型 | 估计行数 | +|------|---------|---------| +| `aperag/api/components/schemas/document.yaml` | 新增 `fetchUrlRequest/Response` schema | ~40 行 | +| `aperag/api/paths/collections.yaml` | 新增 `/fetch-url` 路径 | ~30 行 | +| `aperag/views/collections.py` | 新增一个路由函数 | ~60 行 | +| `aperag/websearch/reader/reader_service.py` | 将私有函数提取为公共方法 | ~20 行重构 | +| `web/src/...import/import-dialog.tsx` | 新建组件 | ~60 行 | +| `web/src/...import/url-import.tsx` | 新建组件 | ~80 行 | +| `web/src/...import/text-import.tsx` | 新建组件 | ~70 行 | +| `web/src/.../document-upload.tsx` | 增加"添加来源"入口触发 | ~10 行 | +| i18n 文件 | 新增翻译 key | ~20 行 | +| **合计** | | **~390 行** | + +**不需要改动的部分（完全复用）**： +- `document_service.upload_document()` — 无改动 +- `document_service.confirm_documents()` — 无改动 +- Celery 任务 / 索引构建流程 — 无改动 +- 文档列表页、暂存区主逻辑 — 几乎无改动 + +--- + +## API Schema 定义 + +在 `aperag/api/components/schemas/document.yaml` 中新增： + +```yaml +fetchUrlRequest: + type: object + properties: + urls: + type: array + items: + type: string + format: uri + minItems: 1 + maxItems: 10 + description: List of URLs to fetch content from + example: + - "https://example.com/article1" + - "https://example.com/article2" + required: + - urls + +fetchUrlResultItem: + type: object + properties: + url: + type: string + description: The source URL + fetch_status: + type: string + enum: ["success", "error"] + document: + $ref: '#/Document' + description: Created document (only present on success) + error: + type: string + description: Error message (only present on failure) + required: + - url + - fetch_status + +fetchUrlResponse: + type: object + properties: + documents: + type: array + items: + $ref: '#/fetchUrlResultItem' + total: + type: integer + succeeded: + type: integer + failed: + type: integer + required: + - documents + - total + - succeeded + - failed +``` + +--- + +## 错误处理 + +| 场景 | 处理位置 | 处理方式 | +|------|---------|---------| +| URL 格式非法（非 http/https） | 后端校验 | 400，跳过该 URL | +| URL 数量 > 10 | 后端校验 | 400，整体拒绝 | +| URL 页面无法访问（4xx/5xx） | web/read 返回 error | 在响应中标记该 URL 失败 | +| 抓取超时（>30s） | web/read 超时 | 同上 | +| 文档名冲突 | `upload_document()` 抛出异常 | 自动追加序号或返回已存在文档（幂等） | +| 配额超限 | `upload_document()` 抛出异常 | 400，停止处理剩余 URL | +| 文本内容为空 | 前端校验 | 禁用"添加"按钮 | + +--- + +## 数据完整性 + +URL 导入的文档在 `doc_metadata` 中记录来源信息，便于追溯和未来的定时刷新功能： + +```json +{ + "source_type": "url", + "source_url": "https://example.com/article", + "page_title": "示例文章标题", + "fetched_at": "2026-03-05T10:00:00Z", + "object_path": "user-xxx/col_xxx/doc_xxx/original.md" +} +``` + +文本导入的文档： + +```json +{ + "source_type": "text", + "object_path": "user-xxx/col_xxx/doc_xxx/original.txt" +} +``` + +--- + +## 与现有功能对比 + +| 维度 | 文件上传 | URL 导入 | 文本导入 | +|------|----------|---------|---------| +| 内容获取方式 | 用户本地文件 | 后端调用 web/read 服务抓取 | 前端直接创建 File 对象 | +| 新增后端接口 | — | 1 个（`/fetch-url`） | **0 个** | +| 初始文档状态 | `UPLOADED` | `UPLOADED` | `UPLOADED` | +| 确认流程 | `POST /documents/confirm` | 同左（完全复用） | 同左（完全复用） | +| 索引构建 | 现有 Celery 任务 | 同左（完全复用） | 同左（完全复用） | +| 文件格式 | 各种格式 | `.md`（Markdown） | `.txt` | + +--- + +## 未来扩展 + +1. **网络搜索导入**：用户输入关键词 → 调用 `web/search` → 获取 URL 列表 → 复用 `/fetch-url` 接口批量抓取。搜索步骤是新增逻辑，抓取和入库完全复用。 + +2. **URL 定时刷新**：对 `source_type=url` 的文档，基于 `source_url` 定期重新抓取内容并更新索引。 + +3. **JavaScript 渲染支持**：web/read 服务升级支持 Playwright 后，`/fetch-url` 接口自动受益，无需改动。 + +--- + +## 实施路径 + +### Phase 1：后端（约 2 天） + +1. 在 `document.yaml` schema 中新增 `fetchUrlRequest/Response` +2. 在 `collections.yaml` paths 中注册 `/fetch-url` 路由 +3. 运行 `make generate-models` +4. 将 `web.py` 中的 `_read_with_jina_fallback` / `_read_with_trafilatura_only` 提取为 `ReaderService` 的公共方法 +5. 在 `views/collections.py` 中实现 `fetch_url_document_view` 路由函数 +6. 编写单元测试 + +### Phase 2：前端（约 2 天） + +1. 运行 `make generate-frontend-sdk` +2. 实现 `import-dialog.tsx`（来源选择入口） +3. 实现 `url-import.tsx`（URL 输入 + 调用 `/fetch-url`） +4. 实现 `text-import.tsx`（文本粘贴 + 创建 File 对象 + 调用现有上传接口） +5. 在文档列表页或上传页集成"添加来源"按钮触发 Dialog +6. 新增 i18n key（`zh-CN` 和 `en-US`） + +### Phase 3：验证（约 0.5 天） + +1. E2E 测试：URL 导入 → 暂存区展示 → confirm → 索引完成 +2. E2E 测试：文本粘贴 → 暂存区展示 → confirm → 索引完成 +3. 错误场景：无效 URL、超时、配额超限 + +--- + +## 相关文件索引 + +### 参考文件 + +- `aperag/service/document_service.py` — `upload_document()` 实现（核心复用点） +- `aperag/views/web.py` — `_read_with_jina_fallback()` 等私有函数（待提取为公共方法） +- `aperag/websearch/reader/reader_service.py` — ReaderService（JINA/Trafilatura 实现） +- `web/src/app/.../upload/document-upload.tsx` — 前端暂存区（参考现有实现） + +### 修改文件 + +- `aperag/api/components/schemas/document.yaml` — 新增 Schema +- `aperag/api/paths/collections.yaml` — 新增路由 +- `aperag/views/collections.py` — 新增 `fetch_url_document_view` +- `aperag/websearch/reader/reader_service.py` — 提取公共方法 + +### 新建文件 + +- `web/src/app/.../documents/upload/import/import-dialog.tsx` +- `web/src/app/.../documents/upload/import/url-import.tsx` +- `web/src/app/.../documents/upload/import/text-import.tsx` +- `web/src/i18n/zh-CN/page_documents_import.json` +- `web/src/i18n/en-US/page_documents_import.json` diff --git a/docs/zh-CN/integration/dify.md b/docs/zh-CN/integration/dify.md index 016022cf3..1108de8ae 100644 --- a/docs/zh-CN/integration/dify.md +++ b/docs/zh-CN/integration/dify.md @@ -31,7 +31,7 @@ ApeRAG 是一款具备多模态索引、AI 智能体、MCP 支持及可扩展 K8 ## Step 1: 准备知识库 -访问 ApeRAG 官网 https://rag.apecloud.com/ ，注册登录后选择或导入一个知识库。这里以三国演义知识库为例，点击订阅知识库。 +打开 ApeRAG Web 界面（见[快速开始](../../../README-zh.md#快速开始)；Docker Compose 启动时一般为 http://localhost:3000/web/）。登录后选择或导入知识库。下文以「三国演义」知识库为例，点击订阅。
订阅知识库 @@ -49,7 +49,7 @@ ApeRAG 是一款具备多模态索引、AI 智能体、MCP 支持及可扩展 K8 ### 2.2 填写配置信息 -填写 Server URL：`https://rag.apecloud.com/mcp/`，以及在 ApeRAG 中复制的 API Key，点击确定。 +填写 Server URL：`http://localhost:8000/mcp/`（若非本机部署，请改为实际 API 地址，例如 `https://<你的域名>/mcp/`），并粘贴从 ApeRAG 复制的 API Key，点击确定。
配置 MCP diff --git a/docs/zh-CN/integration/mcp-api.md b/docs/zh-CN/integration/mcp-api.md index 9a1a96cf3..bb1ebe54b 100644 --- a/docs/zh-CN/integration/mcp-api.md +++ b/docs/zh-CN/integration/mcp-api.md @@ -17,7 +17,7 @@ ApeRAG 通过 [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) { "mcpServers": { "aperag": { - "url": "https://rag.apecloud.com/mcp/", + "url": "http://localhost:8000/mcp/", "headers": { "Authorization": "Bearer your-api-key-here" } diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/test_audit_decorator.py b/tests/unit/test_audit_decorator.py new file mode 100644 index 000000000..dc50c7612 --- /dev/null +++ b/tests/unit/test_audit_decorator.py @@ -0,0 +1,276 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the @audit decorator. + +We verify that: +- start_time and end_time are recorded and passed to audit_service.log_audit +- duration is always non-negative +- successful calls are recorded with status_code=200 +- failed calls are recorded with status_code=500 and the exception is re-raised +- GET requests are skipped (audit decorator does not log GETs) +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from starlette.datastructures import Headers +from starlette.requests import Request +from starlette.testclient import TestClient + +from aperag.utils.audit_decorator import audit + + +# --------------------------------------------------------------------------- +# Minimal fake Request helpers +# --------------------------------------------------------------------------- + + +def _make_request(method: str = "POST", path: str = "/api/v1/bots") -> Request: + """Build a minimal Starlette Request object suitable for the decorator.""" + scope = { + "type": "http", + "method": method, + "path": path, + "query_string": b"", + "headers": [], + "state": {}, + } + request = Request(scope) + # The decorator reads user_id / username from request.state + request.state.user_id = "user-42" + request.state.username = "tester" + return request + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_patched_audit_service(): + """Return a mock for audit_service.log_audit that captures call args.""" + mock_log = AsyncMock() + return mock_log + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_audit_records_start_and_end_time(): + """start_time and end_time must be forwarded to audit_service.log_audit.""" + mock_log = _make_patched_audit_service() + + @audit(resource_type="bot", api_name="CreateBot") + async def _view(request, **kwargs): + return {"id": "bot-1"} + + request = _make_request("POST") + + with patch("aperag.utils.audit_decorator.audit_service.log_audit", mock_log): + await _view(request=request) + + # asyncio.create_task wraps the coroutine; we need to flush the event loop. + # In pytest-asyncio with asyncio_mode=auto the event loop runs between awaits, + # but create_task schedules a new task. We can inspect the mock directly since + # create_task is called synchronously and the coroutine arg contains our args. + assert mock_log.called or True # create_task schedules it; see below + + # Instead, capture via direct call instead of create_task. Patch create_task too. + import asyncio + + captured_coros = [] + + def _capture_task(coro): + captured_coros.append(coro) + # Return a real task so the event loop doesn't complain + return asyncio.ensure_future(coro) + + with patch("asyncio.create_task", side_effect=_capture_task): + await _view(request=request) + + # Wait for all captured coroutines + import asyncio as _asyncio + for coro in captured_coros: + try: + await coro + except Exception: + pass + + assert mock_log.called + kwargs = mock_log.call_args.kwargs + assert kwargs["start_time"] is not None + assert kwargs["end_time"] is not None + assert kwargs["end_time"] >= kwargs["start_time"] + + +@pytest.mark.asyncio +async def test_audit_duration_is_non_negative(): + """end_time - start_time must always be >= 0.""" + import asyncio + + mock_log = _make_patched_audit_service() + captured_coros = [] + + def _capture_task(coro): + captured_coros.append(coro) + return asyncio.ensure_future(coro) + + @audit(resource_type="bot", api_name="CreateBot") + async def _view(request, **kwargs): + return {"ok": True} + + request = _make_request("POST") + + with ( + patch("aperag.utils.audit_decorator.audit_service.log_audit", mock_log), + patch("asyncio.create_task", side_effect=_capture_task), + ): + await _view(request=request) + + for coro in captured_coros: + try: + await coro + except Exception: + pass + + kwargs = mock_log.call_args.kwargs + assert kwargs["end_time"] - kwargs["start_time"] >= 0 + + +@pytest.mark.asyncio +async def test_audit_success_uses_status_200(): + """Successful calls must be audited with status_code=200.""" + import asyncio + + mock_log = _make_patched_audit_service() + captured_coros = [] + + def _capture_task(coro): + captured_coros.append(coro) + return asyncio.ensure_future(coro) + + @audit(resource_type="collection", api_name="CreateCollection") + async def _view(request, **kwargs): + return {"id": "col-1"} + + request = _make_request("POST", "/api/v1/collections") + + with ( + patch("aperag.utils.audit_decorator.audit_service.log_audit", mock_log), + patch("asyncio.create_task", side_effect=_capture_task), + ): + result = await _view(request=request) + + for coro in captured_coros: + try: + await coro + except Exception: + pass + + assert result == {"id": "col-1"} + kwargs = mock_log.call_args.kwargs + assert kwargs["status_code"] == 200 + assert kwargs["error_message"] is None + + +@pytest.mark.asyncio +async def test_audit_failure_uses_status_500_and_reraises(): + """Failed calls must be audited with status_code=500, and the exception re-raised.""" + import asyncio + + mock_log = _make_patched_audit_service() + captured_coros = [] + + def _capture_task(coro): + captured_coros.append(coro) + return asyncio.ensure_future(coro) + + @audit(resource_type="bot", api_name="CreateBot") + async def _view(request, **kwargs): + raise ValueError("something went wrong") + + request = _make_request("POST") + + with ( + patch("aperag.utils.audit_decorator.audit_service.log_audit", mock_log), + patch("asyncio.create_task", side_effect=_capture_task), + ): + with pytest.raises(ValueError, match="something went wrong"): + await _view(request=request) + + for coro in captured_coros: + try: + await coro + except Exception: + pass + + kwargs = mock_log.call_args.kwargs + assert kwargs["status_code"] == 500 + assert kwargs["error_message"] == "something went wrong" + + +@pytest.mark.asyncio +async def test_audit_skips_get_requests(): + """GET requests must be passed through without any audit log.""" + mock_log = _make_patched_audit_service() + + @audit(resource_type="bot", api_name="GetBot") + async def _view(request, **kwargs): + return {"id": "bot-1"} + + request = _make_request("GET") + + with patch("aperag.utils.audit_decorator.audit_service.log_audit", mock_log): + result = await _view(request=request) + + assert result == {"id": "bot-1"} + mock_log.assert_not_called() + + +@pytest.mark.asyncio +async def test_audit_api_name_defaults_to_function_name(): + """If api_name is omitted, the function name should be used.""" + import asyncio + + mock_log = _make_patched_audit_service() + captured_coros = [] + + def _capture_task(coro): + captured_coros.append(coro) + return asyncio.ensure_future(coro) + + @audit(resource_type="bot") + async def create_bot_view(request, **kwargs): + return {} + + request = _make_request("POST") + + with ( + patch("aperag.utils.audit_decorator.audit_service.log_audit", mock_log), + patch("asyncio.create_task", side_effect=_capture_task), + ): + await create_bot_view(request=request) + + for coro in captured_coros: + try: + await coro + except Exception: + pass + + kwargs = mock_log.call_args.kwargs + assert kwargs["api_name"] == "create_bot_view" diff --git a/tests/unit/test_audit_service.py b/tests/unit/test_audit_service.py new file mode 100644 index 000000000..d7d5c5960 --- /dev/null +++ b/tests/unit/test_audit_service.py @@ -0,0 +1,247 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for AuditService. + +All DB interactions are mocked; these tests do *not* require a running +database. +""" + +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from aperag.db.models import AuditLog, AuditResource +from aperag.service.audit_service import AuditService + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_service() -> AuditService: + return AuditService() + + +def _make_mock_session(): + """Return an async context-manager mock that acts like an AsyncSession.""" + session = MagicMock() + session.add = MagicMock() + session.commit = AsyncMock() + + @asynccontextmanager + async def _ctx(): + yield session + + return _ctx(), session + + +# --------------------------------------------------------------------------- +# log_audit — duration_ms computation +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_log_audit_duration_ms_computed_and_stored(): + """duration_ms must equal end_time - start_time and be saved on the row.""" + service = _make_service() + + start_time = 1_700_000_000_000 # milliseconds epoch + end_time = start_time + 250 + expected_duration = 250 + + saved_rows = [] + + ctx, mock_session = _make_mock_session() + + def _capture_add(row): + saved_rows.append(row) + + mock_session.add.side_effect = _capture_add + + with patch.object(service, "_make_session", return_value=ctx): + await service.log_audit( + user_id="u1", + username="alice", + resource_type=AuditResource.BOT, + api_name="CreateBot", + http_method="POST", + path="/api/v1/bots", + status_code=200, + start_time=start_time, + end_time=end_time, + ) + + assert len(saved_rows) == 1 + row: AuditLog = saved_rows[0] + assert row.duration_ms == expected_duration + assert row.start_time == start_time + assert row.end_time == end_time + + +@pytest.mark.asyncio +async def test_log_audit_duration_ms_none_when_end_time_missing(): + """If end_time is None, duration_ms should remain None (not crash).""" + service = _make_service() + + saved_rows = [] + ctx, mock_session = _make_mock_session() + mock_session.add.side_effect = lambda row: saved_rows.append(row) + + with patch.object(service, "_make_session", return_value=ctx): + await service.log_audit( + user_id="u1", + username="alice", + resource_type=AuditResource.BOT, + api_name="CreateBot", + http_method="POST", + path="/api/v1/bots", + status_code=200, + start_time=1_700_000_000_000, + end_time=None, + ) + + assert len(saved_rows) == 1 + assert saved_rows[0].duration_ms is None + + +@pytest.mark.asyncio +async def test_log_audit_session_committed(): + """session.commit() must be called exactly once per log_audit call.""" + service = _make_service() + + ctx, mock_session = _make_mock_session() + + with patch.object(service, "_make_session", return_value=ctx): + await service.log_audit( + user_id="u1", + username="alice", + resource_type=AuditResource.COLLECTION, + api_name="CreateCollection", + http_method="POST", + path="/api/v1/collections", + status_code=200, + start_time=1_700_000_000_000, + end_time=1_700_000_000_100, + ) + + mock_session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_log_audit_does_not_raise_on_db_error(): + """A DB failure in log_audit must be swallowed (fire-and-forget semantics).""" + service = _make_service() + + @asynccontextmanager + async def _failing_ctx(): + raise RuntimeError("DB unavailable") + yield # noqa: unreachable + + with patch.object(service, "_make_session", return_value=_failing_ctx()): + # Should not raise + await service.log_audit( + user_id="u1", + username="alice", + resource_type=AuditResource.BOT, + api_name="CreateBot", + http_method="POST", + path="/api/v1/bots", + status_code=200, + start_time=1_700_000_000_000, + end_time=1_700_000_000_200, + ) + + +# --------------------------------------------------------------------------- +# list_audit_logs — duration_ms back-fill +# --------------------------------------------------------------------------- + + +def _make_audit_log_row(duration_ms=None, start_time=None, end_time=None): + """Create a minimal row-like object without touching the DB. + + We use SimpleNamespace rather than an uninitialised SQLAlchemy model + instance because SA instrumentation requires _sa_instance_state to be + present before column attributes can be set. + """ + from types import SimpleNamespace + + return SimpleNamespace( + duration_ms=duration_ms, + start_time=start_time, + end_time=end_time, + resource_type=None, + path=None, + resource_id=None, + ) + + +@pytest.mark.asyncio +async def test_list_audit_logs_backfills_duration_ms_when_null(): + """Rows with duration_ms=NULL should have it filled from start/end_time.""" + service = _make_service() + + start = 1_700_000_000_000 + end = start + 500 + row = _make_audit_log_row(duration_ms=None, start_time=start, end_time=end) + + # Patch _make_session and paginate_query so no DB is needed + ctx, mock_session = _make_mock_session() + + with ( + patch.object(service, "_make_session", return_value=ctx), + patch( + "aperag.utils.pagination.PaginationHelper.paginate_query", + new=AsyncMock(return_value=([row], 1)), + ), + patch( + "aperag.utils.pagination.PaginationHelper.build_response", + return_value={"items": [row], "total": 1}, + ), + ): + result = await service.list_audit_logs() + + # After processing, row.duration_ms should be back-filled + assert row.duration_ms == 500 + + +@pytest.mark.asyncio +async def test_list_audit_logs_does_not_overwrite_existing_duration_ms(): + """Rows that already have duration_ms set must not be overwritten.""" + service = _make_service() + + start = 1_700_000_000_000 + end = start + 500 + row = _make_audit_log_row(duration_ms=42, start_time=start, end_time=end) + + ctx, mock_session = _make_mock_session() + + with ( + patch.object(service, "_make_session", return_value=ctx), + patch( + "aperag.utils.pagination.PaginationHelper.paginate_query", + new=AsyncMock(return_value=([row], 1)), + ), + patch( + "aperag.utils.pagination.PaginationHelper.build_response", + return_value={"items": [row], "total": 1}, + ), + ): + await service.list_audit_logs() + + # Original value must be unchanged + assert row.duration_ms == 42 diff --git a/tests/unit/test_latency_middleware.py b/tests/unit/test_latency_middleware.py new file mode 100644 index 000000000..893d98255 --- /dev/null +++ b/tests/unit/test_latency_middleware.py @@ -0,0 +1,136 @@ +# Copyright 2025 ApeCloud, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for LatencyLoggingMiddleware. + +These tests wrap a minimal Starlette/ASGI app with the middleware and use +httpx's ASGITransport so no real network socket is needed. +""" + +import logging + +import pytest +from httpx import ASGITransport, AsyncClient +from starlette.applications import Starlette +from starlette.responses import PlainTextResponse +from starlette.routing import Route + +from aperag.middleware.latency import _SKIP_PATHS, LatencyLoggingMiddleware + + +# --------------------------------------------------------------------------- +# Minimal test app +# --------------------------------------------------------------------------- + +def _hello(request): + return PlainTextResponse("hello world") + + +def _health(request): + return PlainTextResponse("ok") + + +_routes = [ + Route("/hello", _hello), + Route("/health", _health), +] + +_app = Starlette(routes=_routes) +_app.add_middleware(LatencyLoggingMiddleware) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +async def _get(path: str): + """Issue a GET to the test app and return the response.""" + async with AsyncClient(transport=ASGITransport(app=_app), base_url="http://testserver") as client: + return await client.get(path) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_x_response_time_header_present(): + """X-Response-Time header must be present on a normal response.""" + resp = await _get("/hello") + assert resp.status_code == 200 + assert "x-response-time" in resp.headers + + +@pytest.mark.asyncio +async def test_x_response_time_header_format(): + """X-Response-Time value must look like 'ms'.""" + resp = await _get("/hello") + value = resp.headers["x-response-time"] + assert value.endswith("ms"), f"Expected '…ms', got {value!r}" + ms_str = value[:-2] + assert ms_str.isdigit(), f"Non-numeric part before 'ms': {ms_str!r}" + assert int(ms_str) >= 0 + + +@pytest.mark.asyncio +async def test_response_body_preserved(): + """The middleware must not alter the response body.""" + resp = await _get("/hello") + assert resp.text == "hello world" + + +@pytest.mark.asyncio +async def test_normal_path_logged_at_info(caplog): + """/hello should produce an INFO log line.""" + with caplog.at_level(logging.INFO, logger="aperag.middleware.latency"): + await _get("/hello") + + info_records = [r for r in caplog.records if r.levelno == logging.INFO] + assert info_records, "Expected at least one INFO log record for /hello" + # The message should contain the path + assert any("/hello" in r.getMessage() for r in info_records) + + +@pytest.mark.asyncio +async def test_skip_path_logged_at_debug_not_info(caplog): + """/health is in _SKIP_PATHS and must be logged at DEBUG, not INFO.""" + assert "/health" in _SKIP_PATHS, "/health should be a skip path" + + with caplog.at_level(logging.DEBUG, logger="aperag.middleware.latency"): + await _get("/health") + + debug_records = [r for r in caplog.records if r.levelno == logging.DEBUG] + info_records = [ + r for r in caplog.records + if r.levelno == logging.INFO and "/health" in r.getMessage() + ] + + assert debug_records, "Expected at least one DEBUG log record for /health" + assert not info_records, "Expected NO INFO log record for a skip path" + + +@pytest.mark.asyncio +async def test_log_line_contains_method_path_status_duration(caplog): + """The INFO log line should contain method, path, status code, and duration.""" + with caplog.at_level(logging.INFO, logger="aperag.middleware.latency"): + await _get("/hello") + + messages = [r.getMessage() for r in caplog.records if r.levelno == logging.INFO] + assert messages, "No INFO log records found" + msg = messages[0] + assert "GET" in msg + assert "/hello" in msg + assert "200" in msg + assert "ms" in msg diff --git a/web/docs/en-US/integration/dify.md b/web/docs/en-US/integration/dify.md index f05e7d201..f6595da66 100644 --- a/web/docs/en-US/integration/dify.md +++ b/web/docs/en-US/integration/dify.md @@ -31,7 +31,7 @@ ApeRAG is a production-grade RAG platform with multimodal indexing, AI agents, M ## Step 1: Prepare Knowledge Base -Visit ApeRAG at https://rag.apecloud.com/ , register/login, and select or import a knowledge base. Here we use the Romance of the Three Kingdoms example - click subscribe. +Open your ApeRAG web UI (see [Quick Start](../../../../README.md#quick-start); with Docker Compose this is typically http://localhost:3000/web/). Sign in and select or import a knowledge base. This walkthrough uses the Romance of the Three Kingdoms example—click **Subscribe**.
Subscribe to Collection @@ -49,7 +49,7 @@ Go to Dify - Tools - MCP, click Add MCP Server. ### 2.2 Fill Configuration -Fill in Server URL: `https://rag.apecloud.com/mcp/` and your API Key copied from ApeRAG, then click Confirm. +Fill in Server URL: `http://localhost:8000/mcp/` (use `https:///mcp/` if ApeRAG is not local), paste your API Key from ApeRAG, then click Confirm.
Configure MCP diff --git a/web/docs/en-US/integration/mcp-api.md b/web/docs/en-US/integration/mcp-api.md index 2c1275b17..85af530ef 100644 --- a/web/docs/en-US/integration/mcp-api.md +++ b/web/docs/en-US/integration/mcp-api.md @@ -17,7 +17,7 @@ For Claude Desktop, add to configuration file: { "mcpServers": { "aperag": { - "url": "https://rag.apecloud.com/mcp/", + "url": "http://localhost:8000/mcp/", "headers": { "Authorization": "Bearer your-api-key-here" } diff --git a/web/docs/zh-CN/integration/dify.md b/web/docs/zh-CN/integration/dify.md index 016022cf3..ca02e13cc 100644 --- a/web/docs/zh-CN/integration/dify.md +++ b/web/docs/zh-CN/integration/dify.md @@ -31,7 +31,7 @@ ApeRAG 是一款具备多模态索引、AI 智能体、MCP 支持及可扩展 K8 ## Step 1: 准备知识库 -访问 ApeRAG 官网 https://rag.apecloud.com/ ，注册登录后选择或导入一个知识库。这里以三国演义知识库为例，点击订阅知识库。 +打开 ApeRAG Web 界面（见[快速开始](../../../../README-zh.md#快速开始)；Docker Compose 启动时一般为 http://localhost:3000/web/）。登录后选择或导入知识库。下文以「三国演义」知识库为例，点击订阅。
订阅知识库 @@ -49,7 +49,7 @@ ApeRAG 是一款具备多模态索引、AI 智能体、MCP 支持及可扩展 K8 ### 2.2 填写配置信息 -填写 Server URL：`https://rag.apecloud.com/mcp/`，以及在 ApeRAG 中复制的 API Key，点击确定。 +填写 Server URL：`http://localhost:8000/mcp/`（若非本机部署，请改为实际 API 地址，例如 `https://<你的域名>/mcp/`），并粘贴从 ApeRAG 复制的 API Key，点击确定。
配置 MCP diff --git a/web/docs/zh-CN/integration/mcp-api.md b/web/docs/zh-CN/integration/mcp-api.md index 9a1a96cf3..bb1ebe54b 100644 --- a/web/docs/zh-CN/integration/mcp-api.md +++ b/web/docs/zh-CN/integration/mcp-api.md @@ -17,7 +17,7 @@ ApeRAG 通过 [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) { "mcpServers": { "aperag": { - "url": "https://rag.apecloud.com/mcp/", + "url": "http://localhost:8000/mcp/", "headers": { "Authorization": "Bearer your-api-key-here" } diff --git a/web/public/chatgpt-ui/index.html b/web/public/chatgpt-ui/index.html new file mode 100644 index 000000000..af2ff1afd --- /dev/null +++ b/web/public/chatgpt-ui/index.html @@ -0,0 +1,517 @@ + + + + + +ChatGPT RAG — ApeRAG + + + + + + +
+ +
+
+

ChatGPT RAG

+

Graph RAG with hybrid retrieval, knowledge graphs, and multi-provider LLM

+
+ + + + +
+
+
+
+
+ + + +
+
+
+ + + + diff --git a/web/src/api/apis/default-api.ts b/web/src/api/apis/default-api.ts index 3e5b6ba26..b7ece9ab2 100644 --- a/web/src/api/apis/default-api.ts +++ b/web/src/api/apis/default-api.ts @@ -96,6 +96,10 @@ import type { FailResponse } from '../models'; // @ts-ignore import type { Feedback } from '../models'; // @ts-ignore +import type { FetchUrlRequest } from '../models'; +// @ts-ignore +import type { FetchUrlResponse } from '../models'; +// @ts-ignore import type { Invitation } from '../models'; // @ts-ignore import type { InvitationCreate } from '../models'; @@ -158,6 +162,8 @@ import type { SharedCollectionList } from '../models'; // @ts-ignore import type { SharingStatusResponse } from '../models'; // @ts-ignore +import type { StagedDocumentsResponse } from '../models'; +// @ts-ignore import type { TagFilterRequest } from '../models'; // @ts-ignore import type { TitleGenerateRequest } from '../models'; @@ -1301,6 +1307,50 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati options: localVarRequestOptions, }; }, + /** + * Fetch web page content from one or more URLs and create UPLOADED documents. Each URL is fetched using the web read service (JINA with Trafilatura fallback). Successfully fetched URLs produce UPLOADED documents in the staging area, identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing. + * @summary Fetch documents from URLs + * @param {string} collectionId + * @param {FetchUrlRequest} fetchUrlRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + collectionsCollectionIdDocumentsFetchUrlPost: async (collectionId: string, fetchUrlRequest: FetchUrlRequest, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'collectionId' is not null or undefined + assertParamExists('collectionsCollectionIdDocumentsFetchUrlPost', 'collectionId', collectionId) + // verify required parameter 'fetchUrlRequest' is not null or undefined + assertParamExists('collectionsCollectionIdDocumentsFetchUrlPost', 'fetchUrlRequest', fetchUrlRequest) + const localVarPath = `/collections/{collection_id}/documents/fetch-url` + .replace(`{${"collection_id"}}`, encodeURIComponent(String(collectionId))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication BearerAuth required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(fetchUrlRequest, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, /** * Get a paginated list of documents with sorting and search capabilities * @summary List documents @@ -1408,6 +1458,44 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati options: localVarRequestOptions, }; }, + /** + * Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation. + * @summary List staged documents + * @param {string} collectionId + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + collectionsCollectionIdDocumentsStagedGet: async (collectionId: string, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'collectionId' is not null or undefined + assertParamExists('collectionsCollectionIdDocumentsStagedGet', 'collectionId', collectionId) + const localVarPath = `/collections/{collection_id}/documents/staged` + .replace(`{${"collection_id"}}`, encodeURIComponent(String(collectionId))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication BearerAuth required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, /** * Upload a single document file to temporary storage (UPLOADED status) * @summary Upload a single document @@ -4089,6 +4177,20 @@ export const DefaultApiFp = function(configuration?: Configuration) { const localVarOperationServerBasePath = operationServerMap['DefaultApi.collectionsCollectionIdDocumentsDocumentIdRebuildIndexesPost']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, + /** + * Fetch web page content from one or more URLs and create UPLOADED documents. Each URL is fetched using the web read service (JINA with Trafilatura fallback). Successfully fetched URLs produce UPLOADED documents in the staging area, identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing. + * @summary Fetch documents from URLs + * @param {string} collectionId + * @param {FetchUrlRequest} fetchUrlRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async collectionsCollectionIdDocumentsFetchUrlPost(collectionId: string, fetchUrlRequest: FetchUrlRequest, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.collectionsCollectionIdDocumentsFetchUrlPost(collectionId, fetchUrlRequest, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.collectionsCollectionIdDocumentsFetchUrlPost']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, /** * Get a paginated list of documents with sorting and search capabilities * @summary List documents @@ -4121,6 +4223,19 @@ export const DefaultApiFp = function(configuration?: Configuration) { const localVarOperationServerBasePath = operationServerMap['DefaultApi.collectionsCollectionIdDocumentsPost']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, + /** + * Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation. + * @summary List staged documents + * @param {string} collectionId + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async collectionsCollectionIdDocumentsStagedGet(collectionId: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.collectionsCollectionIdDocumentsStagedGet(collectionId, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['DefaultApi.collectionsCollectionIdDocumentsStagedGet']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, /** * Upload a single document file to temporary storage (UPLOADED status) * @summary Upload a single document @@ -5205,6 +5320,16 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa collectionsCollectionIdDocumentsDocumentIdRebuildIndexesPost(requestParameters: DefaultApiCollectionsCollectionIdDocumentsDocumentIdRebuildIndexesPostRequest, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.collectionsCollectionIdDocumentsDocumentIdRebuildIndexesPost(requestParameters.collectionId, requestParameters.documentId, requestParameters.rebuildIndexesRequest, options).then((request) => request(axios, basePath)); }, + /** + * Fetch web page content from one or more URLs and create UPLOADED documents. Each URL is fetched using the web read service (JINA with Trafilatura fallback). Successfully fetched URLs produce UPLOADED documents in the staging area, identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing. + * @summary Fetch documents from URLs + * @param {DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + collectionsCollectionIdDocumentsFetchUrlPost(requestParameters: DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.collectionsCollectionIdDocumentsFetchUrlPost(requestParameters.collectionId, requestParameters.fetchUrlRequest, options).then((request) => request(axios, basePath)); + }, /** * Get a paginated list of documents with sorting and search capabilities * @summary List documents @@ -5225,6 +5350,16 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa collectionsCollectionIdDocumentsPost(requestParameters: DefaultApiCollectionsCollectionIdDocumentsPostRequest, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.collectionsCollectionIdDocumentsPost(requestParameters.collectionId, requestParameters.documentCreate, options).then((request) => request(axios, basePath)); }, + /** + * Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation. + * @summary List staged documents + * @param {DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + collectionsCollectionIdDocumentsStagedGet(requestParameters: DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.collectionsCollectionIdDocumentsStagedGet(requestParameters.collectionId, options).then((request) => request(axios, basePath)); + }, /** * Upload a single document file to temporary storage (UPLOADED status) * @summary Upload a single document @@ -6095,6 +6230,16 @@ export interface DefaultApiInterface { */ collectionsCollectionIdDocumentsDocumentIdRebuildIndexesPost(requestParameters: DefaultApiCollectionsCollectionIdDocumentsDocumentIdRebuildIndexesPostRequest, options?: RawAxiosRequestConfig): AxiosPromise; + /** + * Fetch web page content from one or more URLs and create UPLOADED documents. Each URL is fetched using the web read service (JINA with Trafilatura fallback). Successfully fetched URLs produce UPLOADED documents in the staging area, identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing. + * @summary Fetch documents from URLs + * @param {DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApiInterface + */ + collectionsCollectionIdDocumentsFetchUrlPost(requestParameters: DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest, options?: RawAxiosRequestConfig): AxiosPromise; + /** * Get a paginated list of documents with sorting and search capabilities * @summary List documents @@ -6115,6 +6260,16 @@ export interface DefaultApiInterface { */ collectionsCollectionIdDocumentsPost(requestParameters: DefaultApiCollectionsCollectionIdDocumentsPostRequest, options?: RawAxiosRequestConfig): AxiosPromise; + /** + * Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation. + * @summary List staged documents + * @param {DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApiInterface + */ + collectionsCollectionIdDocumentsStagedGet(requestParameters: DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest, options?: RawAxiosRequestConfig): AxiosPromise; + /** * Upload a single document file to temporary storage (UPLOADED status) * @summary Upload a single document @@ -7227,6 +7382,27 @@ export interface DefaultApiCollectionsCollectionIdDocumentsDocumentIdRebuildInde readonly rebuildIndexesRequest: RebuildIndexesRequest } +/** + * Request parameters for collectionsCollectionIdDocumentsFetchUrlPost operation in DefaultApi. + * @export + * @interface DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest + */ +export interface DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest { + /** + * + * @type {string} + * @memberof DefaultApiCollectionsCollectionIdDocumentsFetchUrlPost + */ + readonly collectionId: string + + /** + * + * @type {FetchUrlRequest} + * @memberof DefaultApiCollectionsCollectionIdDocumentsFetchUrlPost + */ + readonly fetchUrlRequest: FetchUrlRequest +} + /** * Request parameters for collectionsCollectionIdDocumentsGet operation in DefaultApi. * @export @@ -7297,6 +7473,20 @@ export interface DefaultApiCollectionsCollectionIdDocumentsPostRequest { readonly documentCreate: DocumentCreate } +/** + * Request parameters for collectionsCollectionIdDocumentsStagedGet operation in DefaultApi. + * @export + * @interface DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest + */ +export interface DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest { + /** + * + * @type {string} + * @memberof DefaultApiCollectionsCollectionIdDocumentsStagedGet + */ + readonly collectionId: string +} + /** * Request parameters for collectionsCollectionIdDocumentsUploadPost operation in DefaultApi. * @export @@ -8586,6 +8776,18 @@ export class DefaultApi extends BaseAPI implements DefaultApiInterface { return DefaultApiFp(this.configuration).collectionsCollectionIdDocumentsDocumentIdRebuildIndexesPost(requestParameters.collectionId, requestParameters.documentId, requestParameters.rebuildIndexesRequest, options).then((request) => request(this.axios, this.basePath)); } + /** + * Fetch web page content from one or more URLs and create UPLOADED documents. Each URL is fetched using the web read service (JINA with Trafilatura fallback). Successfully fetched URLs produce UPLOADED documents in the staging area, identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing. + * @summary Fetch documents from URLs + * @param {DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public collectionsCollectionIdDocumentsFetchUrlPost(requestParameters: DefaultApiCollectionsCollectionIdDocumentsFetchUrlPostRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).collectionsCollectionIdDocumentsFetchUrlPost(requestParameters.collectionId, requestParameters.fetchUrlRequest, options).then((request) => request(this.axios, this.basePath)); + } + /** * Get a paginated list of documents with sorting and search capabilities * @summary List documents @@ -8610,6 +8812,18 @@ export class DefaultApi extends BaseAPI implements DefaultApiInterface { return DefaultApiFp(this.configuration).collectionsCollectionIdDocumentsPost(requestParameters.collectionId, requestParameters.documentCreate, options).then((request) => request(this.axios, this.basePath)); } + /** + * Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation. + * @summary List staged documents + * @param {DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public collectionsCollectionIdDocumentsStagedGet(requestParameters: DefaultApiCollectionsCollectionIdDocumentsStagedGetRequest, options?: RawAxiosRequestConfig) { + return DefaultApiFp(this.configuration).collectionsCollectionIdDocumentsStagedGet(requestParameters.collectionId, options).then((request) => request(this.axios, this.basePath)); + } + /** * Upload a single document file to temporary storage (UPLOADED status) * @summary Upload a single document diff --git a/web/src/api/models/fetch-url-request.ts b/web/src/api/models/fetch-url-request.ts new file mode 100644 index 000000000..8c8b654f7 --- /dev/null +++ b/web/src/api/models/fetch-url-request.ts @@ -0,0 +1,30 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * ApeRAG API + * ApeRAG API Documentation + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface FetchUrlRequest + */ +export interface FetchUrlRequest { + /** + * List of URLs to fetch and import (max 10) + * @type {Array} + * @memberof FetchUrlRequest + */ + 'urls': Array; +} + diff --git a/web/src/api/models/fetch-url-response.ts b/web/src/api/models/fetch-url-response.ts new file mode 100644 index 000000000..3fd5155e0 --- /dev/null +++ b/web/src/api/models/fetch-url-response.ts @@ -0,0 +1,51 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * ApeRAG API + * ApeRAG API Documentation + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { FetchUrlResultItem } from './fetch-url-result-item'; + +/** + * + * @export + * @interface FetchUrlResponse + */ +export interface FetchUrlResponse { + /** + * Results for each URL + * @type {Array} + * @memberof FetchUrlResponse + */ + 'results': Array; + /** + * Total number of URLs processed + * @type {number} + * @memberof FetchUrlResponse + */ + 'total': number; + /** + * Number of URLs successfully fetched + * @type {number} + * @memberof FetchUrlResponse + */ + 'succeeded': number; + /** + * Number of URLs that failed + * @type {number} + * @memberof FetchUrlResponse + */ + 'failed': number; +} + diff --git a/web/src/api/models/fetch-url-result-item.ts b/web/src/api/models/fetch-url-result-item.ts new file mode 100644 index 000000000..495940cb3 --- /dev/null +++ b/web/src/api/models/fetch-url-result-item.ts @@ -0,0 +1,74 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * ApeRAG API + * ApeRAG API Documentation + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * + * @export + * @interface FetchUrlResultItem + */ +export interface FetchUrlResultItem { + /** + * The source URL + * @type {string} + * @memberof FetchUrlResultItem + */ + 'url': string; + /** + * Whether the URL was fetched successfully + * @type {string} + * @memberof FetchUrlResultItem + */ + 'fetch_status': FetchUrlResultItemFetchStatusEnum; + /** + * ID of the created document (only present on success) + * @type {string} + * @memberof FetchUrlResultItem + */ + 'document_id'?: string; + /** + * Filename of the created document (only present on success) + * @type {string} + * @memberof FetchUrlResultItem + */ + 'filename'?: string; + /** + * Size of the created document in bytes (only present on success) + * @type {number} + * @memberof FetchUrlResultItem + */ + 'size'?: number; + /** + * Document status (only present on success) + * @type {string} + * @memberof FetchUrlResultItem + */ + 'status'?: string; + /** + * Error message (only present on failure) + * @type {string} + * @memberof FetchUrlResultItem + */ + 'error'?: string; +} + +export const FetchUrlResultItemFetchStatusEnum = { + success: 'success', + error: 'error' +} as const; + +export type FetchUrlResultItemFetchStatusEnum = typeof FetchUrlResultItemFetchStatusEnum[keyof typeof FetchUrlResultItemFetchStatusEnum]; + + diff --git a/web/src/api/models/index.ts b/web/src/api/models/index.ts index bd0fbe267..af5da806d 100644 --- a/web/src/api/models/index.ts +++ b/web/src/api/models/index.ts @@ -78,6 +78,9 @@ export * from './execution-config-retry'; export * from './export-task-response'; export * from './fail-response'; export * from './feedback'; +export * from './fetch-url-request'; +export * from './fetch-url-response'; +export * from './fetch-url-result-item'; export * from './fulltext-search-params'; export * from './graph-edge'; export * from './graph-edge-properties'; @@ -166,6 +169,7 @@ export * from './shared-collection'; export * from './shared-collection-config'; export * from './shared-collection-list'; export * from './sharing-status-response'; +export * from './staged-documents-response'; export * from './suggestion-action-request'; export * from './suggestion-action-response'; export * from './summary-search-params'; diff --git a/web/src/api/models/staged-documents-response.ts b/web/src/api/models/staged-documents-response.ts new file mode 100644 index 000000000..6f3a032bd --- /dev/null +++ b/web/src/api/models/staged-documents-response.ts @@ -0,0 +1,39 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * ApeRAG API + * ApeRAG API Documentation + * + * The version of the OpenAPI document: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { UploadDocumentResponse } from './upload-document-response'; + +/** + * + * @export + * @interface StagedDocumentsResponse + */ +export interface StagedDocumentsResponse { + /** + * List of staged (UPLOADED) documents awaiting confirmation + * @type {Array} + * @memberof StagedDocumentsResponse + */ + 'documents': Array; + /** + * Total number of staged documents + * @type {number} + * @memberof StagedDocumentsResponse + */ + 'total': number; +} + diff --git a/web/src/api/openapi.merged.yaml b/web/src/api/openapi.merged.yaml index 4cda93dda..e4838e94e 100644 --- a/web/src/api/openapi.merged.yaml +++ b/web/src/api/openapi.merged.yaml @@ -1185,6 +1185,96 @@ paths: application/json: schema: $ref: '#/components/schemas/failResponse' + /collections/{collection_id}/documents/fetch-url: + post: + summary: Fetch documents from URLs + description: | + Fetch web page content from one or more URLs and create UPLOADED documents. + Each URL is fetched using the web read service (JINA with Trafilatura fallback). + Successfully fetched URLs produce UPLOADED documents in the staging area, + identical to file uploads. Use the confirm endpoint to move them to PENDING and start indexing. + security: + - BearerAuth: [] + parameters: + - name: collection_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/fetchUrlRequest' + examples: + single_url: + summary: Single URL + value: + urls: + - https://example.com/article + multiple_urls: + summary: Multiple URLs + value: + urls: + - https://example.com/article1 + - https://example.com/article2 + responses: + '200': + description: URL fetch completed (partial success is also 200) + content: + application/json: + schema: + $ref: '#/components/schemas/fetchUrlResponse' + '400': + description: Bad request - invalid URLs or too many URLs + content: + application/json: + schema: + $ref: '#/components/schemas/failResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/failResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/failResponse' + /collections/{collection_id}/documents/staged: + get: + summary: List staged documents + description: Returns all UPLOADED (staged) documents for the collection that are awaiting confirmation. + security: + - BearerAuth: [] + parameters: + - name: collection_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Staged documents retrieved successfully + content: + application/json: + schema: + $ref: '#/components/schemas/stagedDocumentsResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/failResponse' + '404': + description: Collection not found + content: + application/json: + schema: + $ref: '#/components/schemas/failResponse' /collections/{collection_id}/searches: get: summary: Get search history @@ -6050,6 +6140,88 @@ components: required: - confirmed_count - failed_count + fetchUrlRequest: + type: object + properties: + urls: + type: array + items: + type: string + format: uri + minItems: 1 + maxItems: 10 + description: List of URLs to fetch and import (max 10) + example: + - https://example.com/article1 + - https://example.com/article2 + required: + - urls + fetchUrlResultItem: + type: object + properties: + url: + type: string + description: The source URL + fetch_status: + type: string + enum: + - success + - error + description: Whether the URL was fetched successfully + document_id: + type: string + description: ID of the created document (only present on success) + filename: + type: string + description: Filename of the created document (only present on success) + size: + type: integer + description: Size of the created document in bytes (only present on success) + status: + type: string + description: Document status (only present on success) + error: + type: string + description: Error message (only present on failure) + required: + - url + - fetch_status + fetchUrlResponse: + type: object + properties: + results: + type: array + items: + $ref: '#/components/schemas/fetchUrlResultItem' + description: Results for each URL + total: + type: integer + description: Total number of URLs processed + succeeded: + type: integer + description: Number of URLs successfully fetched + failed: + type: integer + description: Number of URLs that failed + required: + - results + - total + - succeeded + - failed + stagedDocumentsResponse: + type: object + properties: + documents: + type: array + items: + $ref: '#/components/schemas/uploadDocumentResponse' + description: List of staged (UPLOADED) documents awaiting confirmation + total: + type: integer + description: Total number of staged documents + required: + - documents + - total vectorSearchParams: type: object properties: diff --git a/web/src/app/marketplace/collections/[collectionId]/graph/page.tsx b/web/src/app/marketplace/collections/[collectionId]/graph/page.tsx index 2769225d1..93fa0c397 100644 --- a/web/src/app/marketplace/collections/[collectionId]/graph/page.tsx +++ b/web/src/app/marketplace/collections/[collectionId]/graph/page.tsx @@ -1,4 +1,4 @@ -import { CollectionGraph } from '@/app/workspace/collections/[collectionId]/graph/collection-graph'; +import { CollectionGraphHybrid } from '@/app/workspace/collections/[collectionId]/graph/collection-graph-hybrid'; import { PageContainer, PageContent } from '@/components/page-container'; import { getServerApi } from '@/lib/api/server'; import { CollectionHeader } from '../collection-header'; @@ -18,10 +18,10 @@ export default async function Page({ return ( -
+
- +
diff --git a/web/src/app/workspace/collections/[collectionId]/documents/upload/document-upload.tsx b/web/src/app/workspace/collections/[collectionId]/documents/upload/document-upload.tsx index fc3549618..36c520dc7 100644 --- a/web/src/app/workspace/collections/[collectionId]/documents/upload/document-upload.tsx +++ b/web/src/app/workspace/collections/[collectionId]/documents/upload/document-upload.tsx @@ -3,8 +3,16 @@ import { UploadDocumentResponseStatusEnum } from '@/api'; import { useCollectionContext } from '@/components/providers/collection-provider'; import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; import async from 'async'; -import { Bs1CircleFill, Bs2CircleFill, Bs3CircleFill } from 'react-icons/bs'; +import { Bs1CircleFill, Bs2CircleFill } from 'react-icons/bs'; +import { TextImport } from './import/text-import'; +import { UrlImport } from './import/url-import'; import { DataGrid, DataGridPagination } from '@/components/data-grid'; import { Checkbox } from '@/components/ui/checkbox'; @@ -37,9 +45,9 @@ import _ from 'lodash'; import { BrushCleaning, ChevronRight, - CloudUpload, EllipsisVertical, - FolderSearch, + FileText, + Globe, LoaderCircle, Save, Trash, @@ -51,14 +59,20 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { defaultStyles, FileIcon } from 'react-file-icon'; import { toast } from 'sonner'; +/** + * A staging-area entry. Two sources: + * 1. Loaded from DB (GET /staged) — `file` is undefined, document_id always set. + * 2. An in-progress file upload — `file` holds the real File object, document_id + * is set once the upload completes. + */ type DocumentsWithFile = { - file: File; + /** Present only for in-progress file uploads. */ + file?: File; + filename: string; + size: number; progress: number; progress_status: 'pending' | 'uploading' | 'success' | 'failed'; - document_id?: string; - filename?: string; - size?: number; status?: UploadDocumentResponseStatusEnum; }; @@ -72,14 +86,96 @@ export const DocumentUpload = () => { const router = useRouter(); const [documents, setDocuments] = useState([]); const [step, setStep] = useState(1); + const [urlDialogOpen, setUrlDialogOpen] = useState(false); + const [textDialogOpen, setTextDialogOpen] = useState(false); const [rowSelection, setRowSelection] = useState({}); const [isUploading, setIsUploading] = useState(false); - const [pagination, setPagination] = useState({ - pageIndex: 0, - pageSize: 20, - }); + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 20 }); const uploadingFilesRef = useRef>(new Set()); + // ── Staged document helpers ────────────────────────────────────────────── + + /** + * Load UPLOADED documents from the DB and merge them with any currently + * in-progress local uploads. DB records are the source of truth for + * completed items; in-progress uploads (no document_id yet) are kept as-is. + */ + const refreshStaged = useCallback(async () => { + if (!collection.id) return; + try { + const res = + await apiClient.defaultApi.collectionsCollectionIdDocumentsStagedGet({ + collectionId: collection.id, + }); + const staged: DocumentsWithFile[] = res.data.documents.map((doc) => ({ + filename: doc.filename, + size: doc.size, + document_id: doc.document_id, + status: doc.status as UploadDocumentResponseStatusEnum, + progress: 100, + progress_status: 'success' as const, + })); + setDocuments((prev) => { + // Keep uploads that are still in progress (no document_id assigned yet) + const inProgress = prev.filter((d) => d.file && !d.document_id); + return [...staged, ...inProgress]; + }); + } catch (err) { + console.error('Failed to load staged documents', err); + } + }, [collection.id]); + + // Load staged documents when the page opens + useEffect(() => { + refreshStaged(); + }, [refreshStaged]); + + // ── Import success callbacks ───────────────────────────────────────────── + + const handleUrlImportSuccess = useCallback( + ( + results: { + url: string; + fetch_status: 'success' | 'error'; + document_id?: string; + filename?: string; + size?: number; + status?: string; + error?: string; + }[], + ) => { + const succeeded = results.filter( + (r) => r.fetch_status === 'success' && r.document_id, + ); + const failed = results.filter((r) => r.fetch_status === 'error'); + + if (succeeded.length > 0) { + toast.success( + page_documents('import_url_success', { + count: String(succeeded.length), + }), + ); + refreshStaged(); + } + if (failed.length > 0) { + toast.error( + page_documents('import_url_partial', { + succeeded: String(succeeded.length), + failed: String(failed.length), + }), + ); + } + }, + [page_documents, refreshStaged], + ); + + const handleTextImportSuccess = useCallback(() => { + toast.success(page_documents('import_text_success')); + refreshStaged(); + }, [page_documents, refreshStaged]); + + // ── Confirm (save to collection) ───────────────────────────────────────── + const handleSaveToCollection = useCallback(async () => { if (!collection.id) return; const res = @@ -97,19 +193,19 @@ export const DocumentUpload = () => { } }, [collection.id, documents, router]); + // ── Upload machinery ───────────────────────────────────────────────────── + const stopUpload = useCallback(() => { setIsUploading(false); uploadController?.abort(); }, []); - /** - * stop upload after page unmount - */ useEffect(() => stopUpload, [stopUpload]); const startUpload = useCallback( (docs: DocumentsWithFile[]) => { const filesToUpload = docs.filter((doc) => { + if (!doc.file) return false; const fileKey = `${doc.file.name}-${doc.file.size}-${doc.file.lastModified}`; return ( doc.progress_status === 'pending' && @@ -121,12 +217,12 @@ export const DocumentUpload = () => { if (filesToUpload.length === 0) return; filesToUpload.forEach((doc) => { - const fileKey = `${doc.file.name}-${doc.file.size}-${doc.file.lastModified}`; + const fileKey = `${doc.file!.name}-${doc.file!.size}-${doc.file!.lastModified}`; uploadingFilesRef.current.add(fileKey); }); const tasks: AsyncTask[] = filesToUpload.map((_doc) => async (callback) => { - const file = _doc.file; + const file = _doc.file!; if (!collection?.id) { callback(); return; @@ -139,11 +235,10 @@ export const DocumentUpload = () => { await new Promise((resolve) => setTimeout(resolve, Math.random() * 5 + 5), ); - // Update progress for this specific file uploadedChunks++; const progress = (uploadedChunks / totalChunks) * 99; setDocuments((docs) => { - const doc = docs.find((doc) => _.isEqual(doc.file, file)); + const doc = docs.find((d) => d.file && _.isEqual(d.file, file)); if (doc) { doc.progress = Number(progress.toFixed(0)); doc.progress_status = 'uploading'; @@ -156,19 +251,14 @@ export const DocumentUpload = () => { try { const [res] = await Promise.all([ apiClient.defaultApi.collectionsCollectionIdDocumentsUploadPost( - { - collectionId: collection.id, - file: _doc.file, - }, - { - timeout: 1000 * 30, - }, + { collectionId: collection.id, file }, + { timeout: 1000 * 30 }, ), networkSimulation(), ]); setDocuments((docs) => { - const doc = docs.find((doc) => _.isEqual(doc.file, file)); + const doc = docs.find((d) => d.file && _.isEqual(d.file, file)); if (doc && res.data.document_id) { Object.assign(doc, { ...res.data, @@ -181,12 +271,9 @@ export const DocumentUpload = () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars } catch (err) { setDocuments((docs) => { - const doc = docs.find((doc) => _.isEqual(doc.file, file)); + const doc = docs.find((d) => d.file && _.isEqual(d.file, file)); if (doc) { - Object.assign(doc, { - progress: 0, - progress_status: 'failed', - }); + Object.assign(doc, { progress: 0, progress_status: 'failed' }); } return [...docs]; }); @@ -211,11 +298,8 @@ export const DocumentUpload = () => { } }, (err) => { - if (err) { - console.error('Error:', err); - } else { - console.log('upload complated'); - } + if (err) console.error('Upload error:', err); + else console.log('Upload completed'); setIsUploading(false); }, ); @@ -223,11 +307,20 @@ export const DocumentUpload = () => { [collection.id], ); - const handleRemoveFile = useCallback((item: DocumentsWithFile) => { - setDocuments((docs) => - docs.filter((doc) => !_.isEqual(doc.file, item.file)), - ); - }, []); + const handleRemoveFile = useCallback( + (item: DocumentsWithFile) => { + setDocuments((docs) => + docs.filter((doc) => + item.file + ? !_.isEqual(doc.file, item.file) + : doc.document_id !== item.document_id, + ), + ); + }, + [], + ); + + // ── DataGrid columns ───────────────────────────────────────────────────── const columns: ColumnDef[] = useMemo( () => [ @@ -261,8 +354,9 @@ export const DocumentUpload = () => { accessorKey: 'filename', header: page_documents('filename'), cell: ({ row }) => { - const file = row.original.file; - const extension = _.last(file.type.split('/')) || ''; + const { filename, file, size } = row.original; + const mimeType = file?.type ?? ''; + const extension = _.last(mimeType.split('/')) || _.last(filename.split('.')) || ''; return (
@@ -273,9 +367,9 @@ export const DocumentUpload = () => { />
-
{file.name}
+
{filename}
- {(row.original.file.size / 1000).toFixed(0) + ' KB'} + {(size / 1000).toFixed(0) + ' KB'}
@@ -285,30 +379,29 @@ export const DocumentUpload = () => { { header: page_documents('file_type'), cell: ({ row }) => { - return row.original.file.type; + const { file, filename } = row.original; + return file?.type ?? _.last(filename.split('.')) ?? '—'; }, }, { header: page_documents('upload_progress'), - cell: ({ row }) => { - return ( -
- -
-
{row.original.progress}%
-
- {row.original.progress_status} -
+ cell: ({ row }) => ( +
+ +
+
{row.original.progress}%
+
+ {row.original.progress_status}
- ); - }, +
+ ), }, { id: 'actions', @@ -342,11 +435,9 @@ export const DocumentUpload = () => { const table = useReactTable({ data: documents, columns, - state: { - rowSelection, - pagination, - }, - getRowId: (row) => String(row.document_id || row.file.name), + state: { rowSelection, pagination }, + getRowId: (row) => + String(row.document_id ?? (row.file ? `${row.file.name}-${row.file.lastModified}` : row.filename)), enableRowSelection: true, onRowSelectionChange: setRowSelection, onPaginationChange: setPagination, @@ -366,34 +457,42 @@ export const DocumentUpload = () => { const onFileValidate = useCallback( (file: File): string | null => { - const doc = documents.some( + const exists = documents.some( (doc) => - doc.file.name === file.name && - doc.file.size === file.size && - doc.file.lastModified === file.lastModified && - doc.file.type === file.type, + doc.filename === file.name && + (doc.file + ? doc.file.size === file.size && + doc.file.lastModified === file.lastModified + : true), ); - if (doc) { - return 'File already exists.'; - } + if (exists) return 'File already exists.'; return null; }, [documents], ); useEffect(() => { - if (documents.length === 0) { - setStep(1); - } else if ( - documents.filter((doc) => doc.progress_status === 'success').length !== - documents.length + if ( + documents.length === 0 || + documents.some((d) => !d.document_id || d.progress_status !== 'success') ) { - setStep(2); + setStep(1); } else { - setStep(3); + setStep(2); } }, [documents]); + const tabBtnClass = cn( + 'flex items-center gap-1.5 rounded-full px-3 py-1.5 text-sm font-medium transition-colors', + 'text-muted-foreground hover:text-foreground hover:bg-accent cursor-pointer', + ); + + // Only real File objects are passed to FileUpload (for its internal dedup) + const realFiles = useMemo( + () => documents.filter((d) => d.file).map((d) => d.file!), + [documents], + ); + return ( <> { maxSize={100 * 1024 * 1024} className="w-full gap-4" accept=".pdf,.doc,.docx,.txt,.md,.ppt,.pptx,.xls,.xlsx" - value={documents.map((f) => f.file)} + value={realFiles} onValueChange={(files) => { const newDocs: DocumentsWithFile[] = []; const newFilesToUpload: DocumentsWithFile[] = []; files.forEach((file) => { - const existingDoc = documents.find((doc) => - _.isEqual(doc.file, file), + const existingDoc = documents.find( + (doc) => doc.file && _.isEqual(doc.file, file), ); - if (existingDoc) { newDocs.push(existingDoc); } else { const newDoc: DocumentsWithFile = { file, + filename: file.name, + size: file.size, progress_status: 'pending', progress: 0, }; @@ -424,7 +524,9 @@ export const DocumentUpload = () => { } }); - setDocuments(newDocs); + // Preserve DB-loaded staged docs; replace in-progress list + const dbDocs = documents.filter((d) => !d.file); + setDocuments([...dbDocs, ...newDocs]); if (newFilesToUpload.length > 0) { startUpload(newFilesToUpload); @@ -435,6 +537,7 @@ export const DocumentUpload = () => { multiple disabled={isUploading} > + {/* Toolbar */}
{ )} > -
{page_documents('upload')}
-
- -
- -
{page_documents('save_to_collection')}
+
{page_documents('add_documents')}
-
- - - +
{documents.length > 0 && ( - + - + )} - {step === 2 && - (isUploading ? ( - - ) : ( - - ))} - {step === 3 && ( + {isUploading ? ( + + ) : ( )}
+ {/* Main content: drop zone or file list */} {documents.length === 0 ? ( - +
@@ -544,7 +614,69 @@ export const DocumentUpload = () => { )} + + {/* Source picker — always anchored at the bottom */} +
+ {documents.length > 0 && ( + + {page_documents('add_more_sources')} + + )} + + + + + +
+ + {/* URL import dialog */} + + + + {page_documents('import_url_title')} + + { + handleUrlImportSuccess(results); + // Only auto-close when every URL succeeded; keep open on partial failure + // so the user can read the error details before dismissing. + const hasFailures = results.some((r) => r.fetch_status === 'error'); + if (!hasFailures) setUrlDialogOpen(false); + }} + /> + + + + {/* Text import dialog */} + + + + {page_documents('import_text_title')} + + { + handleTextImportSuccess(); + setTextDialogOpen(false); + }} + /> + + ); }; diff --git a/web/src/app/workspace/collections/[collectionId]/documents/upload/import/text-import.tsx b/web/src/app/workspace/collections/[collectionId]/documents/upload/import/text-import.tsx new file mode 100644 index 000000000..6ffc69255 --- /dev/null +++ b/web/src/app/workspace/collections/[collectionId]/documents/upload/import/text-import.tsx @@ -0,0 +1,93 @@ +'use client'; + +import { useCollectionContext } from '@/components/providers/collection-provider'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { Textarea } from '@/components/ui/textarea'; +import { apiClient } from '@/lib/api/client'; +import { LoaderCircle, Type } from 'lucide-react'; +import { useTranslations } from 'next-intl'; +import { useCallback, useState } from 'react'; + +type Props = { + onSuccess: () => void; +}; + +export const TextImport = ({ onSuccess }: Props) => { + const { collection } = useCollectionContext(); + const t = useTranslations('page_documents'); + const [title, setTitle] = useState(''); + const [content, setContent] = useState(''); + const [isUploading, setIsUploading] = useState(false); + + const handleAdd = useCallback(async () => { + if (!collection.id || !content.trim()) return; + + setIsUploading(true); + try { + const filename = title.trim() + ? `${title.trim().slice(0, 200)}.txt` + : `note-${Date.now()}.txt`; + + // Create a File object from the text — reuses the existing upload endpoint entirely + const file = new File([content], filename, { type: 'text/plain' }); + + await apiClient.defaultApi.collectionsCollectionIdDocumentsUploadPost({ + collectionId: collection.id, + file, + }); + + onSuccess(); + } finally { + setIsUploading(false); + } + }, [collection.id, title, content, onSuccess]); + + return ( +
+

{t('import_text_desc')}

+ +
+ + setTitle(e.target.value)} + disabled={isUploading} + maxLength={200} + /> +
+ +
+ +