From 0ed26736061b6d43c52090be8da48201e43e083b Mon Sep 17 00:00:00 2001 From: pucawo Date: Mon, 20 Apr 2026 14:25:52 +0200 Subject: [PATCH 1/4] feat(files): auto-embed chat-input text-file uploads via rag-api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream v0.8.2 only routes file uploads through rag-api when the user explicitly chose an Agent with the file_search tool. Normal chat-input paperclip uploads of PDFs, DOCX, etc. never get indexed — even when RAG_API_URL is configured — so chat-time retrieval (createContextHandlers) has nothing to query against. This patch adds a minimal auto-embed pass in processAgentFileUpload's "standard single storage" branch: for chat-input paperclip uploads (message_file === true, no tool_resource) with a text-bearing MIME type, fire-and-forget uploadVectors to rag-api after the primary storage write. On success, the File doc is flagged embedded:true so the existing createContextHandlers pipeline picks it up at chat time. On any rag-api failure, the upload still succeeds with embedded:false (graceful degradation — user sees the inline attachment). - New: api/server/services/Files/VectorDB/autoEmbed.js - isTextBearingMimeType(mimetype) - tryEmbedChatAttachment({req, file, file_id, entity_id}) — never throws - New: api/server/services/Files/VectorDB/autoEmbed.test.js - MIME classification, success + error + unknown-type paths - Modified: api/server/services/Files/process.js - Standard-storage branch now invokes tryEmbedChatAttachment for messageAttachment && !tool_resource && text-bearing && RAG_API_URL Images (→ Vision) and audio (→ STT) are intentionally skipped. No schema change: fileSchema.embedded already exists upstream. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../services/Files/VectorDB/autoEmbed.js | 58 +++++++++ .../services/Files/VectorDB/autoEmbed.test.js | 117 ++++++++++++++++++ api/server/services/Files/process.js | 34 +++++ 3 files changed, 209 insertions(+) create mode 100644 api/server/services/Files/VectorDB/autoEmbed.js create mode 100644 api/server/services/Files/VectorDB/autoEmbed.test.js diff --git a/api/server/services/Files/VectorDB/autoEmbed.js b/api/server/services/Files/VectorDB/autoEmbed.js new file mode 100644 index 00000000000..a8bf460ea52 --- /dev/null +++ b/api/server/services/Files/VectorDB/autoEmbed.js @@ -0,0 +1,58 @@ +const { logger } = require('@librechat/data-schemas'); +const { uploadVectors } = require('./crud'); + +const TEXT_BEARING_MIME_RE = + /^(application\/(pdf|json|xml|x-sh|x-tar|typescript|sql|yaml|epub\+zip|vnd\.coffeescript|msword|vnd\.ms-(word|powerpoint|excel)|vnd\.openxmlformats-officedocument\.(wordprocessingml\.document|presentationml\.presentation|spreadsheetml\.sheet))|text\/[\w.+-]+)$/; + +const EXCLUDED_PREFIXES = ['image/', 'audio/', 'video/']; + +/** + * Whether a MIME type should be auto-embedded via rag-api for chat attachments. + * Excludes images (go to Vision), audio (go to STT), and video. + * + * @param {string | undefined | null} mimetype + * @returns {boolean} + */ +function isTextBearingMimeType(mimetype) { + if (!mimetype || typeof mimetype !== 'string') { + return false; + } + if (EXCLUDED_PREFIXES.some((prefix) => mimetype.startsWith(prefix))) { + return false; + } + return TEXT_BEARING_MIME_RE.test(mimetype); +} + +/** + * Wrap uploadVectors with graceful degradation: never throws, returns { embedded: false } + * on any failure. Intended for auto-RAG of chat-input paperclip attachments where a + * failed embed must still allow the upload to succeed (user sees the attachment, just + * without indexed-retrieval superpowers). + * + * @param {object} params + * @param {import('express').Request} params.req + * @param {Express.Multer.File} params.file + * @param {string} params.file_id + * @param {string} [params.entity_id] + * @returns {Promise<{ embedded: boolean }>} + */ +async function tryEmbedChatAttachment({ req, file, file_id, entity_id }) { + if (!process.env.RAG_API_URL) { + return { embedded: false }; + } + + try { + const result = await uploadVectors({ req, file, file_id, entity_id }); + return { embedded: Boolean(result?.embedded) }; + } catch (error) { + logger.warn( + `[tryEmbedChatAttachment] rag-api embed failed for file_id=${file_id}, falling back to inline attachment: ${error?.message || error}`, + ); + return { embedded: false }; + } +} + +module.exports = { + isTextBearingMimeType, + tryEmbedChatAttachment, +}; diff --git a/api/server/services/Files/VectorDB/autoEmbed.test.js b/api/server/services/Files/VectorDB/autoEmbed.test.js new file mode 100644 index 00000000000..8370f8756d7 --- /dev/null +++ b/api/server/services/Files/VectorDB/autoEmbed.test.js @@ -0,0 +1,117 @@ +jest.mock('./crud', () => ({ + uploadVectors: jest.fn(), +})); + +jest.mock('@librechat/data-schemas', () => ({ + logger: { warn: jest.fn(), info: jest.fn(), error: jest.fn(), debug: jest.fn() }, +})); + +const { uploadVectors } = require('./crud'); +const { logger } = require('@librechat/data-schemas'); +const { isTextBearingMimeType, tryEmbedChatAttachment } = require('./autoEmbed'); + +describe('VectorDB/autoEmbed', () => { + const originalEnv = process.env.RAG_API_URL; + + afterEach(() => { + jest.clearAllMocks(); + }); + + afterAll(() => { + if (originalEnv === undefined) { + delete process.env.RAG_API_URL; + } else { + process.env.RAG_API_URL = originalEnv; + } + }); + + describe('isTextBearingMimeType', () => { + test.each([ + ['application/pdf', true], + ['application/vnd.openxmlformats-officedocument.wordprocessingml.document', true], + ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', true], + ['application/vnd.openxmlformats-officedocument.presentationml.presentation', true], + ['application/msword', true], + ['application/vnd.ms-excel', true], + ['application/vnd.ms-powerpoint', true], + ['text/plain', true], + ['text/markdown', true], + ['text/csv', true], + ['text/html', true], + ['application/json', true], + ['application/xml', true], + ['text/x-python', true], + ['application/x-sh', true], + ['application/epub+zip', true], + ])('returns true for text-bearing type %s', (mimetype, expected) => { + expect(isTextBearingMimeType(mimetype)).toBe(expected); + }); + + test.each([ + ['image/png', false], + ['image/jpeg', false], + ['image/gif', false], + ['image/webp', false], + ['image/svg+xml', false], + ['audio/mp3', false], + ['audio/mpeg', false], + ['audio/wav', false], + ['video/mp4', false], + ['video/webm', false], + ['', false], + [undefined, false], + [null, false], + ])('returns false for non-text type %s', (mimetype, expected) => { + expect(isTextBearingMimeType(mimetype)).toBe(expected); + }); + }); + + describe('tryEmbedChatAttachment', () => { + const req = { user: { id: 'user-1' } }; + const file = { path: '/tmp/test.pdf', mimetype: 'application/pdf', size: 100, originalname: 'test.pdf' }; + const baseParams = { req, file, file_id: 'fid-1', entity_id: 'conv-1' }; + + test('returns embedded:false when RAG_API_URL not configured', async () => { + delete process.env.RAG_API_URL; + + const result = await tryEmbedChatAttachment(baseParams); + + expect(result).toEqual({ embedded: false }); + expect(uploadVectors).not.toHaveBeenCalled(); + }); + + test('returns embedded:true on successful embed', async () => { + process.env.RAG_API_URL = 'http://rag-api'; + uploadVectors.mockResolvedValueOnce({ embedded: true, bytes: 100 }); + + const result = await tryEmbedChatAttachment(baseParams); + + expect(result).toEqual({ embedded: true }); + expect(uploadVectors).toHaveBeenCalledWith({ + req, + file, + file_id: 'fid-1', + entity_id: 'conv-1', + }); + }); + + test('returns embedded:false on rag-api error without throwing (graceful degradation)', async () => { + process.env.RAG_API_URL = 'http://rag-api'; + uploadVectors.mockRejectedValueOnce(new Error('ECONNREFUSED')); + + const result = await tryEmbedChatAttachment(baseParams); + + expect(result).toEqual({ embedded: false }); + expect(logger.warn).toHaveBeenCalled(); + }); + + test('returns embedded:false when rag-api reports unknown type', async () => { + process.env.RAG_API_URL = 'http://rag-api'; + uploadVectors.mockResolvedValueOnce({ embedded: false, bytes: 100 }); + + const result = await tryEmbedChatAttachment(baseParams); + + expect(result).toEqual({ embedded: false }); + }); + }); +}); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 30b47f2e52b..83a630ed8e7 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -642,6 +642,36 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { basePath, entity_id, }); + + /** + * Auto-RAG for chat-input paperclip uploads: when a user drops a text-bearing + * file into the chat input (message_file === true, no tool_resource), also + * index it in the vector DB so chat-time retrieval works without the user + * having to opt in via an Agent. Images (→ Vision) and audio (→ STT) are + * intentionally skipped. rag-api failures must not fail the upload. + * + * Upstream v0.8.2 never embeds chat-paperclip uploads — only Agent + * file_search and tool_resource=context paths do. + */ + if ( + messageAttachment && + !tool_resource && + process.env.RAG_API_URL && + !isImageFile + ) { + const { isTextBearingMimeType, tryEmbedChatAttachment } = require('./VectorDB/autoEmbed'); + if (isTextBearingMimeType(file.mimetype)) { + const embedResult = await tryEmbedChatAttachment({ + req, + file, + file_id, + entity_id, + }); + if (embedResult.embedded) { + embeddingResult = embedResult; + } + } + } } let { bytes, filename, filepath: _filepath, height, width } = storageResult; @@ -650,6 +680,10 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { if (tool_resource === EToolResources.file_search) { embedded = embeddingResult?.embedded; filename = embeddingResult?.filename || filename; + } else if (embeddingResult?.embedded) { + // Chat-paperclip auto-RAG: flag the file as embedded so chat-time + // createContextHandlers triggers retrieval for it. + embedded = true; } let filepath = _filepath; From 02b8a5bc40feac651107a72e09948527e2622017 Mon Sep 17 00:00:00 2001 From: pucawo Date: Tue, 21 Apr 2026 15:28:42 +0200 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20optional=20multimodal=20retrieval?= =?UTF-8?q?=20=E2=80=94=20attach=20page=20images=20to=20vision=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends createContextHandlers so that when the rag-api fork returns visual matches (multimodal-ingest branch), we: 1. Pass `include_visual: true` on the /query body, controlled by RAG_INCLUDE_VISUAL (default on). 2. Accept both the legacy flat-list /query response and the new `{chunks, visual_matches}` shape, so mixed-version deploys don't break. 3. For vision-capable models (validateVisionModel), base64-encode each matching page PNG and surface it as an image_url the caller can attach to the latest user message. 4. For text-only models, append a short German hint to the RAG context so the LLM doesn't claim to have seen the layout. The AgentClient's non-agent RAG branch merges getVisualImageURLs() onto `latestMessage.image_urls` right after createContext(). This keeps the attachment flow provider-agnostic — Claude / OpenAI adapters handle the final shape transformation downstream, same as for paperclip-uploaded images. Defense-in-depth: rejects any image_path that isn't absolute so a compromised sidecar cannot nudge us into reading arbitrary files. Tests: 7 new jest cases (no RAG_API_URL guard, include_visual propagation, vision vs text-only branching, base64 encoding, path traversal rejection, legacy response compat, feature flag off). --- .../clients/prompts/createContextHandlers.js | 135 +++++++++- .../prompts/createContextHandlers.test.js | 242 ++++++++++++++++++ api/server/controllers/agents/client.js | 18 ++ 3 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 api/app/clients/prompts/createContextHandlers.test.js diff --git a/api/app/clients/prompts/createContextHandlers.js b/api/app/clients/prompts/createContextHandlers.js index 87c48bcf92e..c6887711017 100644 --- a/api/app/clients/prompts/createContextHandlers.js +++ b/api/app/clients/prompts/createContextHandlers.js @@ -1,6 +1,9 @@ const axios = require('axios'); +const fs = require('fs'); +const path = require('path'); const { logger } = require('@librechat/data-schemas'); const { isEnabled, generateShortLivedToken } = require('@librechat/api'); +const { validateVisionModel } = require('librechat-data-provider'); const footer = `Use the context as your learned knowledge to better answer the user. @@ -10,6 +13,63 @@ In your response, remember to follow these guidelines: - Avoid mentioning that you obtained the information from the context. `; +// Multimodal-RAG: visual matches we collected from rag-api. Exposed back to +// the caller via getVisualImageURLs() so client.js can attach them as +// image_urls on the latest message when the model supports vision. +// +// Soft-fail policy matches the rest of this file: if anything throws (axios, +// fs.readFile, missing validateVisionModel) we log a warning and continue +// with the text-only path. + +/** + * Determine whether the currently-selected model can accept image inputs. + * We look up the model name from req.body.model (what the user picked in + * the UI) and fall back to req.endpointOption.model. If neither is set we + * assume no — safer to skip attachments than to crash the provider call. + */ +function resolveIsVisionModel(req) { + const model = req?.body?.model || req?.endpointOption?.model; + if (!model) { + return false; + } + try { + return validateVisionModel({ model }); + } catch (err) { + logger.warn('[createContextHandlers] validateVisionModel threw, assuming no', err); + return false; + } +} + +async function loadVisualImageURLsFromDisk(visualMatches) { + // Small helper so the test can mock just this bit. Reads each page PNG from + // disk and base64-encodes it. Matches the shape that encodeAndFormat / + // LibreChat's provider adapters expect for inline image parts. + const image_urls = []; + for (const match of visualMatches) { + try { + // Sanity: image_path must be an absolute path — rag-api writes them + // as `/var/rag-visual//page-N.png`. Reject anything else to + // avoid accidental path traversal from a compromised sidecar. + if (!path.isAbsolute(match.image_path)) { + continue; + } + const buf = await fs.promises.readFile(match.image_path); + image_urls.push({ + type: 'image_url', + image_url: { + url: `data:image/png;base64,${buf.toString('base64')}`, + detail: 'auto', + }, + }); + } catch (err) { + logger.warn( + `[createContextHandlers] visual attachment: cannot read ${match.image_path}: ${err.message}`, + ); + } + } + return image_urls; +} + function createContextHandlers(req, userMessageContent) { if (!process.env.RAG_API_URL) { return; @@ -20,6 +80,13 @@ function createContextHandlers(req, userMessageContent) { const processedIds = new Set(); const jwtToken = generateShortLivedToken(req.user.id); const useFullContext = isEnabled(process.env.RAG_USE_FULL_CONTEXT); + const multimodalEnabled = isEnabled(process.env.RAG_INCLUDE_VISUAL ?? 'true'); + const isVisionModel = resolveIsVisionModel(req); + + // Collected across all /query responses. One entry per visual match across + // all files the user attached to this message. + /** @type {Array<{file_id:string,page_number:number,image_path:string,score:number}>} */ + const visualMatches = []; const query = async (file) => { if (useFullContext) { @@ -36,6 +103,7 @@ function createContextHandlers(req, userMessageContent) { file_id: file.file_id, query: userMessageContent, k: 4, + include_visual: multimodalEnabled, }, { headers: { @@ -59,6 +127,25 @@ function createContextHandlers(req, userMessageContent) { } }; + /** + * Normalise the /query response. Old rag-api returns a flat list of + * [Document, score] tuples; the multimodal-ingest fork returns + * `{ chunks: [...], visual_matches: [...] }` when include_visual is true. + * We accept both so we never break when rag-api is older than LibreChat. + */ + const normaliseQueryData = (data) => { + if (Array.isArray(data)) { + return { chunks: data, visual_matches: [] }; + } + if (data && typeof data === 'object') { + return { + chunks: Array.isArray(data.chunks) ? data.chunks : [], + visual_matches: Array.isArray(data.visual_matches) ? data.visual_matches : [], + }; + } + return { chunks: [], visual_matches: [] }; + }; + const createContext = async () => { try { if (!queryPromises.length || !processedFiles.length) { @@ -112,7 +199,12 @@ function createContextHandlers(req, userMessageContent) { return generateContext(`\n${contextItems}`); } - contextItems = queryResult.data + const { chunks, visual_matches } = normaliseQueryData(queryResult.data); + if (visual_matches.length) { + visualMatches.push(...visual_matches); + } + + contextItems = chunks .map((item) => { const pageContent = item[0].page_content; return ` @@ -134,6 +226,16 @@ function createContextHandlers(req, userMessageContent) { return prompt; } + // Multimodal-RAG: when visual hits exist but the current model can + // only read text, nudge the LLM to acknowledge rather than claim it + // inspected the page visually. + const visualHint = + multimodalEnabled && visualMatches.length && !isVisionModel + ? `\n[Hinweis: ${visualMatches.length} visuelle Treffer auf Seiten ${visualMatches + .map((v) => v.page_number) + .join(', ')} verfügbar, aber das aktuell gewählte Modell kann keine Bilder auswerten — Antwort basiert nur auf dem Text-Kontext.]` + : ''; + const prompt = `${header} ${files} @@ -141,7 +243,7 @@ function createContextHandlers(req, userMessageContent) { ${context} - + ${visualHint} ${footer}`; return prompt; @@ -151,10 +253,39 @@ function createContextHandlers(req, userMessageContent) { } }; + /** + * Returns the collected visual matches (populated by createContext). + * Callers that support vision inputs can convert these to image_urls + * via getVisualImageURLs(). Must be called after createContext(). + */ + const getVisualMatches = () => [...visualMatches]; + + /** + * Load page PNGs from disk and return them in the image_urls shape the + * downstream provider adapters expect. Only returns entries when the + * current model is vision-capable; otherwise the hint in createContext + * covers the text-only case. + */ + const getVisualImageURLs = async () => { + if (!multimodalEnabled || !isVisionModel || visualMatches.length === 0) { + return []; + } + try { + return await loadVisualImageURLsFromDisk(visualMatches); + } catch (err) { + logger.warn('[createContextHandlers] could not load visual image urls:', err); + return []; + } + }; + return { processFile, createContext, + getVisualMatches, + getVisualImageURLs, }; } module.exports = createContextHandlers; +// Exposed for unit tests only. +module.exports.__test__ = { loadVisualImageURLsFromDisk, resolveIsVisionModel }; diff --git a/api/app/clients/prompts/createContextHandlers.test.js b/api/app/clients/prompts/createContextHandlers.test.js new file mode 100644 index 00000000000..99c6a208bee --- /dev/null +++ b/api/app/clients/prompts/createContextHandlers.test.js @@ -0,0 +1,242 @@ +/** + * Tests for the multimodal-RAG additions to createContextHandlers. + * + * We mock axios (rag-api), fs.promises.readFile (page PNGs) and + * @librechat/api / librechat-data-provider so the module loads without + * requiring the full monorepo wiring — same pattern as autoEmbed.test.js. + */ + +jest.mock('axios'); + +jest.mock( + '@librechat/data-schemas', + () => ({ + logger: { + warn: jest.fn(), + info: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, + }), + { virtual: true }, +); + +jest.mock( + '@librechat/api', + () => ({ + isEnabled: (v) => { + if (v === undefined || v === null) return false; + if (typeof v === 'boolean') return v; + return String(v).toLowerCase() === 'true'; + }, + generateShortLivedToken: jest.fn().mockReturnValue('test-jwt'), + }), + { virtual: true }, +); + +jest.mock( + 'librechat-data-provider', + () => ({ + validateVisionModel: jest.fn(), + }), + { virtual: true }, +); + +const axios = require('axios'); +const fs = require('fs'); +const { validateVisionModel } = require('librechat-data-provider'); + +const createContextHandlers = require('./createContextHandlers'); + +const buildReq = (model) => ({ + user: { id: 'user-1' }, + body: { model }, +}); + +const makeFile = (overrides = {}) => ({ + file_id: 'f-1', + filename: 'flyer.pdf', + type: 'application/pdf', + embedded: true, + ...overrides, +}); + +describe('createContextHandlers — multimodal-RAG', () => { + const originalRagUrl = process.env.RAG_API_URL; + const originalFlag = process.env.RAG_INCLUDE_VISUAL; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.RAG_API_URL = 'http://rag.test'; + delete process.env.RAG_INCLUDE_VISUAL; // default -> treated as 'true' via ?? in prod + delete process.env.RAG_USE_FULL_CONTEXT; + }); + + afterAll(() => { + if (originalRagUrl === undefined) { + delete process.env.RAG_API_URL; + } else { + process.env.RAG_API_URL = originalRagUrl; + } + if (originalFlag === undefined) { + delete process.env.RAG_INCLUDE_VISUAL; + } else { + process.env.RAG_INCLUDE_VISUAL = originalFlag; + } + }); + + test('returns undefined when RAG_API_URL is missing', () => { + delete process.env.RAG_API_URL; + expect(createContextHandlers(buildReq('gpt-4o'), 'hi')).toBeUndefined(); + }); + + test('sends include_visual: true on /query and surfaces visual matches on createContext', async () => { + validateVisionModel.mockReturnValue(true); + + axios.post.mockResolvedValue({ + data: { + chunks: [[{ page_content: 'kleingedrucktes' }, 0.88]], + visual_matches: [ + { + file_id: 'f-1', + page_number: 2, + image_path: '/var/rag-visual/f-1/page-2.png', + score: 0.73, + }, + ], + }, + }); + + const handlers = createContextHandlers(buildReq('gpt-4o'), 'Wie sieht Seite 2 aus?'); + await handlers.processFile(makeFile()); + const prompt = await handlers.createContext(); + + expect(axios.post).toHaveBeenCalledWith( + 'http://rag.test/query', + expect.objectContaining({ include_visual: true }), + expect.any(Object), + ); + expect(prompt).toContain('kleingedrucktes'); + expect(prompt).not.toContain('Hinweis:'); // vision model — no text-only fallback hint + expect(handlers.getVisualMatches()).toHaveLength(1); + expect(handlers.getVisualMatches()[0].page_number).toBe(2); + }); + + test('does not attach image_urls for non-vision models and adds a hint to the prompt', async () => { + validateVisionModel.mockReturnValue(false); + + axios.post.mockResolvedValue({ + data: { + chunks: [[{ page_content: 'text' }, 0.5]], + visual_matches: [ + { + file_id: 'f-1', + page_number: 3, + image_path: '/var/rag-visual/f-1/page-3.png', + score: 0.61, + }, + ], + }, + }); + + const handlers = createContextHandlers(buildReq('some-text-only-model'), 'q'); + await handlers.processFile(makeFile()); + const prompt = await handlers.createContext(); + + expect(prompt).toContain('Hinweis:'); + expect(prompt).toContain('Seiten 3'); + + const urls = await handlers.getVisualImageURLs(); + expect(urls).toEqual([]); + }); + + test('getVisualImageURLs base64-encodes each page PNG for vision models', async () => { + validateVisionModel.mockReturnValue(true); + const readSpy = jest + .spyOn(fs.promises, 'readFile') + .mockResolvedValue(Buffer.from([0x89, 0x50, 0x4e, 0x47])); // PNG magic bytes + + axios.post.mockResolvedValue({ + data: { + chunks: [], + visual_matches: [ + { file_id: 'f-1', page_number: 1, image_path: '/var/rag-visual/f-1/page-1.png', score: 0.9 }, + { file_id: 'f-1', page_number: 2, image_path: '/var/rag-visual/f-1/page-2.png', score: 0.8 }, + ], + }, + }); + + const handlers = createContextHandlers(buildReq('claude-sonnet-4-6'), 'q'); + await handlers.processFile(makeFile()); + await handlers.createContext(); + + const urls = await handlers.getVisualImageURLs(); + expect(urls).toHaveLength(2); + expect(urls[0]).toEqual({ + type: 'image_url', + image_url: { + url: 'data:image/png;base64,iVBORw==', + detail: 'auto', + }, + }); + expect(readSpy).toHaveBeenCalledTimes(2); + readSpy.mockRestore(); + }); + + test('skips non-absolute image_paths (defense against compromised sidecar)', async () => { + validateVisionModel.mockReturnValue(true); + const readSpy = jest.spyOn(fs.promises, 'readFile'); + + axios.post.mockResolvedValue({ + data: { + chunks: [], + visual_matches: [ + { file_id: 'f-1', page_number: 1, image_path: '../escape/etc/passwd', score: 0.9 }, + ], + }, + }); + + const handlers = createContextHandlers(buildReq('gpt-4o'), 'q'); + await handlers.processFile(makeFile()); + await handlers.createContext(); + + const urls = await handlers.getVisualImageURLs(); + expect(urls).toEqual([]); + expect(readSpy).not.toHaveBeenCalled(); + readSpy.mockRestore(); + }); + + test('tolerates legacy flat-list /query response (no visual_matches key)', async () => { + validateVisionModel.mockReturnValue(true); + axios.post.mockResolvedValue({ + data: [[{ page_content: 'legacy content' }, 0.7]], + }); + + const handlers = createContextHandlers(buildReq('gpt-4o'), 'q'); + await handlers.processFile(makeFile()); + const prompt = await handlers.createContext(); + + expect(prompt).toContain('legacy content'); + expect(handlers.getVisualMatches()).toEqual([]); + await expect(handlers.getVisualImageURLs()).resolves.toEqual([]); + }); + + test('multimodalEnabled=false passes include_visual: false and yields no matches', async () => { + process.env.RAG_INCLUDE_VISUAL = 'false'; + validateVisionModel.mockReturnValue(true); + axios.post.mockResolvedValue({ + data: [[{ page_content: 'plain text' }, 0.5]], + }); + + const handlers = createContextHandlers(buildReq('gpt-4o'), 'q'); + await handlers.processFile(makeFile()); + await handlers.createContext(); + + expect(axios.post).toHaveBeenCalledWith( + 'http://rag.test/query', + expect.objectContaining({ include_visual: false }), + expect.any(Object), + ); + await expect(handlers.getVisualImageURLs()).resolves.toEqual([]); + }); +}); diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 90e9640d5c4..367ddc50813 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -488,6 +488,24 @@ class AgentClient extends BaseClient { if (this.augmentedPrompt) { sharedRunContextParts.push(this.augmentedPrompt); } + + // Multimodal-RAG: append page-image hits from rag-api as inline + // image_urls on the latest user message when the selected model + // can read images. Vision-capability + feature-flag gates live in + // createContextHandlers; here we just merge the resulting urls. + if (typeof this.contextHandlers.getVisualImageURLs === 'function') { + try { + const visualImageURLs = await this.contextHandlers.getVisualImageURLs(); + if (visualImageURLs && visualImageURLs.length && latestMessage) { + latestMessage.image_urls = [ + ...(latestMessage.image_urls ?? []), + ...visualImageURLs, + ]; + } + } catch (err) { + logger.warn('[AgentClient] multimodal visual attachment failed:', err); + } + } } /** Memory context (user preferences/memories) */ From cd5655bb8cb424e40ce0967717a61787248f0d48 Mon Sep 17 00:00:00 2001 From: pucawo Date: Tue, 21 Apr 2026 17:48:11 +0200 Subject: [PATCH 3/4] feat: env-configurable VISION_MODEL_ALIASES for custom model names --- api/app/clients/prompts/createContextHandlers.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/app/clients/prompts/createContextHandlers.js b/api/app/clients/prompts/createContextHandlers.js index c6887711017..67737924b81 100644 --- a/api/app/clients/prompts/createContextHandlers.js +++ b/api/app/clients/prompts/createContextHandlers.js @@ -33,7 +33,11 @@ function resolveIsVisionModel(req) { return false; } try { - return validateVisionModel({ model }); + const extra = (process.env.VISION_MODEL_ALIASES || '') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + return validateVisionModel({ model, additionalModels: extra }); } catch (err) { logger.warn('[createContextHandlers] validateVisionModel threw, assuming no', err); return false; From 3118a6cb4b239f960f7b516ba35e10a071e6f76b Mon Sep 17 00:00:00 2001 From: pucawo Date: Thu, 23 Apr 2026 07:36:27 +0200 Subject: [PATCH 4/4] fix(multimodal-retrieval): mirror visualImageURLs onto formattedMessages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The earlier patch attached page PNGs to latestMessage.image_urls, but formattedMessages — built via orderedMessages.map(formatMessage) earlier in the same buildMessages call — holds independent objects, not refs. The provider payload is constructed from formattedMessages downstream, so the mutation on latestMessage never reached the LLM. Vision-capable models received text context only and (correctly) reported they had no images to analyse. Mirror the image_urls assignment onto the matching formattedMessages entry. End-to-end verified on prod with a Claude Opus chat: page PNGs now arrive at the Anthropic API and the model can analyse them. Co-Authored-By: Claude Opus 4.7 (1M context) --- api/server/controllers/agents/client.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 367ddc50813..4589c663fc4 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -501,6 +501,19 @@ class AgentClient extends BaseClient { ...(latestMessage.image_urls ?? []), ...visualImageURLs, ]; + // formattedMessages was .map'd from orderedMessages BEFORE this point and + // holds independent objects, so mutating latestMessage doesn't propagate. + // Mirror the image_urls onto the corresponding formatted entry so the + // provider payload (built from formattedMessages downstream) actually + // carries the page PNGs. Without this, getVisualImageURLs() loads the + // images into memory but they never reach the LLM. + const lastFormatted = formattedMessages[formattedMessages.length - 1]; + if (lastFormatted) { + lastFormatted.image_urls = [ + ...(lastFormatted.image_urls ?? []), + ...visualImageURLs, + ]; + } } } catch (err) { logger.warn('[AgentClient] multimodal visual attachment failed:', err);