From 6fd3479eb2debe7aacd02bc042446ebe98beb798 Mon Sep 17 00:00:00 2001 From: Nocturnal Date: Fri, 21 Aug 2026 10:54:52 +0700 Subject: [PATCH] fix(claude-code): send pasted images to the model as native image blocks The claude-code driver built stdin as a single text block per message, so pasted image attachments never reached the model. They were saved to the attachments sidecar but silently dropped on the way to the brain, and the assistant replied as if the message were text-only. build_stdin now rehydrates `[Image: ... #att:]` placeholders to on-disk markers and emits each as a native Anthropic `image` content block (base64), which the `claude` CLI accepts and Opus can see. An image that cannot be read degrades to a short text note rather than being dropped, and plain-text turns are unaffected. Closes #5649 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../provider/claude_code/input_builder.rs | 94 ++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/src/openhuman/inference/provider/claude_code/input_builder.rs b/src/openhuman/inference/provider/claude_code/input_builder.rs index 462c0d670c..f59a223c93 100644 --- a/src/openhuman/inference/provider/claude_code/input_builder.rs +++ b/src/openhuman/inference/provider/claude_code/input_builder.rs @@ -11,13 +11,21 @@ //! - On a `--resume` of an existing CC session: claude already has prior //! turns server-side; we only send the last user turn. +use base64::Engine as _; use serde_json::{json, Value}; use crate::openhuman::agent::messages::ChatMessage; +use crate::openhuman::agent::multimodal::{parse_image_markers, rehydrate_image_placeholders}; /// Build the bytes to write to claude's stdin. Returns an empty `Vec` /// when there is nothing to send (caller should abort). pub fn build_stdin(messages: &[ChatMessage], is_new_session: bool) -> Vec { + // Resolve any `[Image: … #att:]` placeholders to on-disk `[IMAGE:]` + // markers so pasted images can be inlined below. No-op for messages that + // carry no image placeholder, so plain text turns are unaffected. + let rehydrated = rehydrate_image_placeholders(messages); + let messages: &[ChatMessage] = &rehydrated; + let mut out = String::new(); let to_emit: Vec<&ChatMessage> = if is_new_session { messages.iter().filter(|m| m.role != "system").collect() @@ -44,7 +52,7 @@ pub fn build_stdin(messages: &[ChatMessage], is_new_session: bool) -> Vec { "type": "user", "message": { "role": role, - "content": [{"type": "text", "text": msg.content}], + "content": content_blocks(&msg.content), }, }); push_json_line(&mut out, &line); @@ -53,6 +61,67 @@ pub fn build_stdin(messages: &[ChatMessage], is_new_session: bool) -> Vec { out.into_bytes() } +/// Split a message's text into stream-json content blocks: the prose as a +/// `text` block, plus one native `image` block per `[IMAGE:]` marker (the +/// `claude` CLI + Opus are vision-capable). An image that cannot be read +/// degrades to a short text note rather than being silently dropped. +fn content_blocks(raw: &str) -> Vec { + let (text, image_refs) = parse_image_markers(raw); + let mut blocks: Vec = Vec::new(); + if !text.is_empty() { + blocks.push(json!({"type": "text", "text": text})); + } + for reference in &image_refs { + match image_block(reference) { + Some(block) => blocks.push(block), + None => blocks.push(json!({ + "type": "text", + "text": "[an attached image could not be read]" + })), + } + } + if blocks.is_empty() { + // Preserve prior behaviour for a genuinely empty message. + blocks.push(json!({"type": "text", "text": raw})); + } + blocks +} + +/// Build an Anthropic `image` content block from an `[IMAGE:]` reference. +/// `` is either a `data:` URI (inline base64) or an on-disk file path (a +/// rehydrated attachment). Returns `None` when the ref cannot be resolved. +fn image_block(reference: &str) -> Option { + let (media_type, data_b64) = if let Some(rest) = reference.strip_prefix("data:") { + let (mime, data) = rest.split_once(";base64,")?; + (mime.to_string(), data.to_string()) + } else { + let bytes = std::fs::read(reference).ok()?; + ( + media_type_from_path(reference), + base64::engine::general_purpose::STANDARD.encode(bytes), + ) + }; + Some(json!({ + "type": "image", + "source": {"type": "base64", "media_type": media_type, "data": data_b64}, + })) +} + +/// Best-effort media type from a file extension. Claude accepts jpeg/png/gif/webp. +fn media_type_from_path(path: &str) -> String { + let lower = path.to_ascii_lowercase(); + if lower.ends_with(".png") { + "image/png" + } else if lower.ends_with(".gif") { + "image/gif" + } else if lower.ends_with(".webp") { + "image/webp" + } else { + "image/jpeg" + } + .to_string() +} + fn push_json_line(buf: &mut String, v: &Value) { buf.push_str(&serde_json::to_string(v).unwrap_or_default()); buf.push('\n'); @@ -107,4 +176,27 @@ mod tests { let bytes = build_stdin(&[], true); assert!(bytes.is_empty()); } + + #[test] + fn user_message_with_image_marker_emits_native_image_block() { + // A rehydrated / inline data-URI marker becomes a real image block, and + // the surrounding prose stays a text block. Regression for pasted images + // being dropped on the way to the claude-code brain. + let m = ChatMessage::user("look at this [IMAGE:data:image/png;base64,QUJD]"); + let s = String::from_utf8(build_stdin(&[m], true)).unwrap(); + assert!(s.contains("\"type\":\"image\""), "image block emitted: {s}"); + assert!(s.contains("\"media_type\":\"image/png\""), "{s}"); + assert!(s.contains("\"data\":\"QUJD\""), "base64 payload preserved: {s}"); + assert!(s.contains("\"text\":\"look at this\""), "prose kept: {s}"); + assert!(!s.contains("[IMAGE:"), "raw marker stripped: {s}"); + } + + #[test] + fn plain_text_still_single_text_block() { + // serde_json sorts object keys, so the block serializes as + // {"text":"hi","type":"text"} — a single text block, no image blocks. + let s = String::from_utf8(build_stdin(&[ChatMessage::user("hi")], true)).unwrap(); + assert!(s.contains("\"content\":[{\"text\":\"hi\",\"type\":\"text\"}]"), "{s}"); + assert!(!s.contains("\"type\":\"image\""), "no image block for plain text: {s}"); + } }