diff --git a/src/openhuman/inference/provider/claude_code/input_builder.rs b/src/openhuman/inference/provider/claude_code/input_builder.rs index 462c0d670c..f59a223c93 100644 --- a/src/openhuman/inference/provider/claude_code/input_builder.rs +++ b/src/openhuman/inference/provider/claude_code/input_builder.rs @@ -11,13 +11,21 @@ //! - On a `--resume` of an existing CC session: claude already has prior //! turns server-side; we only send the last user turn. +use base64::Engine as _; use serde_json::{json, Value}; use crate::openhuman::agent::messages::ChatMessage; +use crate::openhuman::agent::multimodal::{parse_image_markers, rehydrate_image_placeholders}; /// Build the bytes to write to claude's stdin. Returns an empty `Vec` /// when there is nothing to send (caller should abort). pub fn build_stdin(messages: &[ChatMessage], is_new_session: bool) -> Vec { + // Resolve any `[Image: … #att:]` placeholders to on-disk `[IMAGE:]` + // markers so pasted images can be inlined below. No-op for messages that + // carry no image placeholder, so plain text turns are unaffected. + let rehydrated = rehydrate_image_placeholders(messages); + let messages: &[ChatMessage] = &rehydrated; + let mut out = String::new(); let to_emit: Vec<&ChatMessage> = if is_new_session { messages.iter().filter(|m| m.role != "system").collect() @@ -44,7 +52,7 @@ pub fn build_stdin(messages: &[ChatMessage], is_new_session: bool) -> Vec { "type": "user", "message": { "role": role, - "content": [{"type": "text", "text": msg.content}], + "content": content_blocks(&msg.content), }, }); push_json_line(&mut out, &line); @@ -53,6 +61,67 @@ pub fn build_stdin(messages: &[ChatMessage], is_new_session: bool) -> Vec { out.into_bytes() } +/// Split a message's text into stream-json content blocks: the prose as a +/// `text` block, plus one native `image` block per `[IMAGE:]` marker (the +/// `claude` CLI + Opus are vision-capable). An image that cannot be read +/// degrades to a short text note rather than being silently dropped. +fn content_blocks(raw: &str) -> Vec { + let (text, image_refs) = parse_image_markers(raw); + let mut blocks: Vec = Vec::new(); + if !text.is_empty() { + blocks.push(json!({"type": "text", "text": text})); + } + for reference in &image_refs { + match image_block(reference) { + Some(block) => blocks.push(block), + None => blocks.push(json!({ + "type": "text", + "text": "[an attached image could not be read]" + })), + } + } + if blocks.is_empty() { + // Preserve prior behaviour for a genuinely empty message. + blocks.push(json!({"type": "text", "text": raw})); + } + blocks +} + +/// Build an Anthropic `image` content block from an `[IMAGE:]` reference. +/// `` is either a `data:` URI (inline base64) or an on-disk file path (a +/// rehydrated attachment). Returns `None` when the ref cannot be resolved. +fn image_block(reference: &str) -> Option { + let (media_type, data_b64) = if let Some(rest) = reference.strip_prefix("data:") { + let (mime, data) = rest.split_once(";base64,")?; + (mime.to_string(), data.to_string()) + } else { + let bytes = std::fs::read(reference).ok()?; + ( + media_type_from_path(reference), + base64::engine::general_purpose::STANDARD.encode(bytes), + ) + }; + Some(json!({ + "type": "image", + "source": {"type": "base64", "media_type": media_type, "data": data_b64}, + })) +} + +/// Best-effort media type from a file extension. Claude accepts jpeg/png/gif/webp. +fn media_type_from_path(path: &str) -> String { + let lower = path.to_ascii_lowercase(); + if lower.ends_with(".png") { + "image/png" + } else if lower.ends_with(".gif") { + "image/gif" + } else if lower.ends_with(".webp") { + "image/webp" + } else { + "image/jpeg" + } + .to_string() +} + fn push_json_line(buf: &mut String, v: &Value) { buf.push_str(&serde_json::to_string(v).unwrap_or_default()); buf.push('\n'); @@ -107,4 +176,27 @@ mod tests { let bytes = build_stdin(&[], true); assert!(bytes.is_empty()); } + + #[test] + fn user_message_with_image_marker_emits_native_image_block() { + // A rehydrated / inline data-URI marker becomes a real image block, and + // the surrounding prose stays a text block. Regression for pasted images + // being dropped on the way to the claude-code brain. + let m = ChatMessage::user("look at this [IMAGE:data:image/png;base64,QUJD]"); + let s = String::from_utf8(build_stdin(&[m], true)).unwrap(); + assert!(s.contains("\"type\":\"image\""), "image block emitted: {s}"); + assert!(s.contains("\"media_type\":\"image/png\""), "{s}"); + assert!(s.contains("\"data\":\"QUJD\""), "base64 payload preserved: {s}"); + assert!(s.contains("\"text\":\"look at this\""), "prose kept: {s}"); + assert!(!s.contains("[IMAGE:"), "raw marker stripped: {s}"); + } + + #[test] + fn plain_text_still_single_text_block() { + // serde_json sorts object keys, so the block serializes as + // {"text":"hi","type":"text"} — a single text block, no image blocks. + let s = String::from_utf8(build_stdin(&[ChatMessage::user("hi")], true)).unwrap(); + assert!(s.contains("\"content\":[{\"text\":\"hi\",\"type\":\"text\"}]"), "{s}"); + assert!(!s.contains("\"type\":\"image\""), "no image block for plain text: {s}"); + } }