Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: add scoped multimodal image attachments

Browse Source
This commit is contained in:
Hare
2026-08-10 21:49:39 +09:00
parent 64ced7dbad
commit 38b8f26a50
42 changed files with 718 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/llm-engine/src/engine.rs
+87 -5
View File
@@ -19,15 +19,19 @@ use crate::{
},
llm_client::{
ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream,
ToolDefinition, error::is_retryable, event::Event, retry::RetryPolicy,
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT, types::parse_tool_arguments,
ToolDefinition,
error::is_retryable,
event::Event,
retry::RetryPolicy,
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT,
types::{ContentPart, parse_tool_arguments},
},
state::{EngineState, Locked, Mutable},
timeline::event::{ErrorEvent, StatusEvent, UsageEvent},
timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector},
tool::{
ToolCall, ToolDefinition as EngineToolDefinition, ToolError, ToolExecutionContext,
ToolOutputLimits, ToolResult, truncate_content,
Attachment, ToolCall, ToolDefinition as EngineToolDefinition, ToolError,
ToolExecutionContext, ToolOutputLimits, ToolResult, truncate_content,
},
tool_server::{ToolServer, ToolServerHandle},
};
@@ -155,6 +159,33 @@ enum StreamCompletion {
Interrupted { reason: String },
}
fn project_transient_attachments(items: &mut Vec<Item>) {
let mut projected = Vec::with_capacity(items.len());
let mut pending_parts = Vec::new();
for mut item in items.drain(..) {
let is_tool_result = matches!(item, Item::ToolResult { .. });
if !is_tool_result && !pending_parts.is_empty() {
projected.push(Item::user_message_parts(std::mem::take(&mut pending_parts)));
}
if let Item::ToolResult { attachments, .. } = &item {
for attachment in attachments {
let Attachment::Image(image) = attachment;
pending_parts.push(ContentPart::image(
image.mime_type(),
Arc::from(image.data()),
));
}
}
item.clear_transient_attachments();
projected.push(item);
}
if !pending_parts.is_empty() {
projected.push(Item::user_message_parts(pending_parts));
}
*items = projected;
}
pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
/// LLM client
client: C,
@@ -1249,6 +1280,8 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
cb(current_llm_call);
}
project_transient_attachments(&mut request_context);
// Stream LLM response
self.emit_lifecycle_trace(
current_turn,
@@ -1257,6 +1290,12 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
items_trace_payload(&request_context, tool_definitions.len(), None, false),
);
let request = self.build_request(&tool_definitions, &request_context);
// Structured attachments are single-use provider payloads. `request`
// owns its Arc-backed copy; history is cleared before any later turn
// can persist or resend the binary body.
for item in &mut self.history {
item.clear_transient_attachments();
}
self.emit_lifecycle_trace(
current_turn,
current_llm_call,
@@ -1618,11 +1657,12 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
// Route per-result pushes through the callback path so
// observers see each tool result as it lands.
let items = results.into_iter().map(|result| {
Item::tool_result_item(
Item::tool_result_item_with_attachments(
&result.tool_use_id,
&result.summary,
result.content,
result.is_error,
result.attachments,
)
});
self.append_history_items(items)?;
@@ -2159,8 +2199,50 @@ fn item_kind(item: &Item) -> &'static str {
#[cfg(test)]
mod tests {
use super::*;
use crate::llm_client::types::Role;
use std::time::Duration;
#[test]
fn transient_tool_attachment_becomes_non_persistent_user_image_part() {
let body: Arc<[u8]> = Arc::from(&b"secret-image-body"[..]);
let mut items = vec![Item::tool_result_item_with_attachments(
"call_image",
"attached",
None,
false,
vec![Attachment::Image(crate::tool::ImageAttachment::new(
"image/png",
body.clone(),
))],
)];
project_transient_attachments(&mut items);
assert!(matches!(
&items[0],
Item::ToolResult { attachments, .. } if attachments.is_empty()
));
match &items[1] {
Item::Message {
role: Role::User,
content,
..
} => match &content[0] {
ContentPart::Image { media_type, source } => {
assert_eq!(media_type, "image/png");
assert_eq!(source.data(), body.as_ref());
assert_eq!(source.bytes(), body.len());
assert_eq!(content[0].as_text(), "[image attachment omitted]");
}
other => panic!("unexpected content part: {other:?}"),
},
other => panic!("unexpected projected item: {other:?}"),
}
let persisted = serde_json::to_string(&items).unwrap();
assert!(!persisted.contains("secret-image-body"));
assert!(!persisted.contains("base64"));
}
#[tokio::test]
async fn first_stream_event_timeout_returns_retryable_timeout() {
let stream: ResponseStream = Box::pin(futures::stream::pending());
crates/llm-engine/src/llm_client/scheme/anthropic/request.rs
+3
View File
@@ -268,6 +268,9 @@ impl AnthropicScheme {
.iter()
.map(|p| match p {
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
ContentPart::Image { .. } => {
AnthropicContentPart::text(p.as_text().to_string())
}
ContentPart::Refusal { refusal } => {
AnthropicContentPart::text(refusal.clone())
}
crates/llm-engine/src/llm_client/scheme/openai_chat/request.rs
+138 -10
View File
@@ -8,7 +8,7 @@ use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{Item, Role, ToolDefinition, parse_tool_arguments},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
};
use super::OpenAIScheme;
@@ -134,7 +134,7 @@ impl OpenAIScheme {
}
// Convert items to messages
messages.extend(self.convert_items_to_messages(&request.items));
messages.extend(self.convert_items_to_messages(&request.items, capability.vision));
let tools = request.tools.iter().map(|t| self.convert_tool(t)).collect();
@@ -185,7 +185,11 @@ impl OpenAIScheme {
/// - Assistant messages have role "assistant"
/// - Tool calls are within assistant messages as tool_calls array
/// - Tool results have role "tool" with tool_call_id
fn convert_items_to_messages(&self, items: &[Item]) -> Vec<OpenAIMessage> {
fn convert_items_to_messages(
&self,
items: &[Item],
supports_images: bool,
) -> Vec<OpenAIMessage> {
let mut messages = Vec::new();
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
let mut pending_assistant_text: Option<String> = None;
@@ -205,16 +209,45 @@ impl OpenAIScheme {
Role::Assistant => "assistant",
Role::System => "system",
};
let text_content: String = content
.iter()
.map(|p| p.as_text())
.collect::<Vec<_>>()
.join("");
let has_image = matches!(role, Role::User)
&& supports_images
&& content
.iter()
.any(|part| matches!(part, ContentPart::Image { .. }));
let message_content = if has_image {
OpenAIContent::Parts(
content
.iter()
.map(|part| match part {
ContentPart::Text { text } => {
OpenAIContentPart::Text { text: text.clone() }
}
ContentPart::Image { media_type, source } => {
OpenAIContentPart::ImageUrl {
image_url: ImageUrl {
url: image_data_url(media_type, source.data()),
},
}
}
ContentPart::Refusal { refusal } => OpenAIContentPart::Text {
text: refusal.clone(),
},
})
.collect(),
)
} else {
OpenAIContent::Text(
content
.iter()
.map(ContentPart::as_text)
.collect::<Vec<_>>()
.join(""),
)
};
messages.push(OpenAIMessage {
role: openai_role.to_string(),
content: Some(OpenAIContent::Text(text_content)),
content: Some(message_content),
tool_calls: vec![],
tool_call_id: None,
name: None,
@@ -334,6 +367,13 @@ mod tests {
}
}
fn vision_cap() -> ModelCapability {
ModelCapability {
vision: true,
..cap()
}
}
#[test]
fn test_build_simple_request() {
let scheme = OpenAIScheme::new();
@@ -439,4 +479,92 @@ mod tests {
assert_eq!(body.messages[1].tool_calls.len(), 1);
assert_eq!(body.messages[2].role, "tool");
}
#[test]
fn parallel_tool_results_precede_synthetic_image_message() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let request = Request::new()
.item(Item::tool_call("call_image", "ViewImage", "{}"))
.item(Item::tool_call("call_text", "Read", "{}"))
.item(Item::tool_result_item(
"call_image",
"Attached image",
None,
false,
))
.item(Item::tool_result_item(
"call_text",
"Read text",
None,
false,
))
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let json = serde_json::to_value(
&scheme
.build_request("gpt-4o", &request, &vision_cap())
.messages,
)
.unwrap();
assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool");
assert_eq!(json[2]["role"], "tool");
assert_eq!(json[3]["role"], "user");
assert_eq!(json[3]["content"][0]["type"], "image_url");
}
#[test]
fn tool_image_is_structured_as_following_user_content_without_persisting_bytes() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
"image/png",
image.clone(),
));
let item = Item::tool_result_item_with_attachments(
"call_image",
"Attached image",
None,
false,
vec![attachment],
);
let persisted = serde_json::to_string(&item).unwrap();
assert!(!persisted.contains("base64"));
assert!(!persisted.contains("attachments"));
let request = Request::new()
.item(Item::tool_call(
"call_image",
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-4o", &request, &vision_cap());
let json = serde_json::to_value(&body.messages).unwrap();
assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool");
assert_eq!(json[2]["role"], "user");
assert_eq!(json[2]["content"][0]["type"], "image_url");
assert!(
json[2]["content"][0]["image_url"]["url"]
.as_str()
.unwrap()
.starts_with("data:image/png;base64,")
);
let mut no_vision = cap();
no_vision.vision = false;
let disabled =
serde_json::to_string(&scheme.build_request("gpt-4o", &request, &no_vision)).unwrap();
assert!(!disabled.contains("data:image"));
}
}
crates/llm-engine/src/llm_client/scheme/openai_responses/request.rs
+65 -12
View File
@@ -10,7 +10,7 @@ use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, parse_tool_arguments},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
};
use super::OpenAIResponsesScheme;
@@ -89,12 +89,7 @@ pub(crate) enum InputItem {
arguments: String,
},
/// function tool の結果（user 側）。
FunctionCallOutput {
call_id: String,
/// Responses は文字列 or 構造化 output を許すが、ここでは
/// `summary` + `content` を改行連結した文字列で送る。
output: String,
},
FunctionCallOutput { call_id: String, output: String },
/// reasoning item。`encrypted_content` があれば必ず添える。
Reasoning {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -118,6 +113,11 @@ pub(crate) enum InputItem {
pub(crate) enum InputContent {
/// user / developer 側のテキスト
InputText { text: String },
/// user 側の画像
InputImage {
image_url: String,
detail: &'static str,
},
/// assistant 側のテキスト
OutputText { text: String },
}
@@ -173,7 +173,7 @@ impl OpenAIResponsesScheme {
request: &Request,
capability: &ModelCapability,
) -> ResponsesRequest {
let input = convert_items_to_input(&request.items);
let input = convert_items_to_input(&request.items, capability.vision);
let tools = request.tools.iter().map(convert_tool).collect();
// Reasoning 投影: capability が Effort / Both をサポートし、かつ
@@ -234,7 +234,7 @@ impl OpenAIResponsesScheme {
}
/// `Item` 列を `input[]` に変換する。
fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputItem> {
let mut out = Vec::with_capacity(items.len());
for item in items {
match item {
@@ -247,8 +247,17 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
};
let parts: Vec<InputContent> = content
.iter()
.map(|p| match p {
.map(|part| match part {
ContentPart::Text { text } => text_variant(text.clone()),
ContentPart::Image { media_type, source }
if matches!(role, Role::User) && supports_images =>
{
InputContent::InputImage {
image_url: image_data_url(media_type, source.data()),
detail: "auto",
}
}
ContentPart::Image { .. } => text_variant(part.as_text().to_string()),
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
})
.collect();
@@ -277,13 +286,13 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
content,
..
} => {
let text = match content {
let output = match content {
Some(c) => format!("{summary}\n{c}"),
None => summary.clone(),
};
out.push(InputItem::FunctionCallOutput {
call_id: call_id.clone(),
output: text,
output,
});
}
Item::Reasoning {
@@ -690,4 +699,48 @@ mod tests {
assert_eq!(json["tools"][0]["type"], "function");
assert_eq!(json["tools"][0]["name"], "t");
}
#[test]
fn synthetic_user_image_uses_responses_input_image_content() {
let scheme = OpenAIResponsesScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let item = Item::tool_result_item_with_attachments(
"call_image",
"Attached image",
None,
false,
vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", image.clone()),
)],
);
let req = Request::new()
.item(Item::tool_call(
"call_image",
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
let json = serde_json::to_value(&body).unwrap();
assert_eq!(json["input"][1]["type"], "function_call_output");
assert_eq!(json["input"][2]["type"], "message");
assert_eq!(json["input"][2]["content"][0]["type"], "input_image");
assert!(
json["input"][2]["content"][0]["image_url"]
.as_str()
.unwrap()
.starts_with("data:image/png;base64,")
);
let mut no_vision = cap_with_reasoning();
no_vision.vision = false;
let disabled =
serde_json::to_string(&scheme.build_request("gpt-5", &req, &no_vision)).unwrap();
assert!(!disabled.contains("data:image"));
}
}
crates/llm-engine/src/llm_client/types.rs
+78 -1
View File
@@ -9,12 +9,19 @@
use std::{fmt, sync::Arc};
use crate::tool::Attachment;
use base64::Engine as _;
use serde::{Deserialize, Serialize};
fn is_false(value: &bool) -> bool {
!*value
}
pub(crate) fn image_data_url(media_type: &str, data: &[u8]) -> String {
let encoded = base64::engine::general_purpose::STANDARD.encode(data);
format!("data:{media_type};base64,{encoded}")
}
// ============================================================================
// Item - The core unit of conversation
// ============================================================================
@@ -117,6 +124,9 @@ pub enum Item {
/// Whether the tool result represents an execution error.
#[serde(default, skip_serializing_if = "is_false")]
is_error: bool,
/// Request-local structured payloads. Never serialized or persisted.
#[serde(skip, default)]
attachments: Vec<Attachment>,
},
/// Reasoning/thinking item
@@ -250,6 +260,17 @@ impl Item {
summary: impl Into<String>,
content: Option<String>,
is_error: bool,
) -> Self {
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
}
/// Create a tool result item with request-local structured attachments.
pub fn tool_result_item_with_attachments(
call_id: impl Into<String>,
summary: impl Into<String>,
content: Option<String>,
is_error: bool,
attachments: Vec<Attachment>,
) -> Self {
Self::ToolResult {
id: None,
@@ -257,6 +278,14 @@ impl Item {
summary: summary.into(),
content,
is_error,
attachments,
}
}
/// Drop request-local attachments after constructing the provider request.
pub fn clear_transient_attachments(&mut self) {
if let Self::ToolResult { attachments, .. } = self {
attachments.clear();
}
}
@@ -428,6 +457,38 @@ pub fn parse_tool_arguments(arguments: &str) -> serde_json::Value {
// Content Parts - Components within message items
// ============================================================================
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ImageSource {
bytes: usize,
#[serde(skip, default)]
data: Arc<[u8]>,
}
impl ImageSource {
pub fn new(data: Arc<[u8]>) -> Self {
Self {
bytes: data.len(),
data,
}
}
pub fn data(&self) -> &[u8] {
&self.data
}
pub fn bytes(&self) -> usize {
self.bytes
}
}
impl fmt::Debug for ImageSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ImageSource")
.field("bytes", &self.bytes)
.finish()
}
}
/// Content part within a message item
///
/// Text content is role-agnostic; the containing Item's Role determines
@@ -441,6 +502,12 @@ pub enum ContentPart {
text: String,
},
/// Request-local image content. The source bytes are never serialized.
Image {
media_type: String,
source: ImageSource,
},
/// Refusal content (for assistant messages)
Refusal {
/// The refusal message
@@ -461,10 +528,20 @@ impl ContentPart {
}
}
/// Get the text content regardless of type
pub fn image(media_type: impl Into<String>, data: Arc<[u8]>) -> Self {
Self::Image {
media_type: media_type.into(),
source: ImageSource::new(data),
}
}
/// Get a bounded textual projection. Image content is represented by an
/// explicit placeholder rather than silently becoming an empty string,
/// filesystem path, data URL, or base64 body.
pub fn as_text(&self) -> &str {
match self {
Self::Text { text } => text,
Self::Image { .. } => "[image attachment omitted]",
Self::Refusal { refusal } => refusal,
}
}
crates/llm-engine/src/tool.rs
+55 -5
View File
@@ -3,8 +3,7 @@
//! Traits for defining tools callable by LLM.
//! Usually auto-implemented using the `#[tool]` macro.
use std::collections::HashMap;
use std::sync::Arc;
use std::{collections::HashMap, fmt, sync::Arc};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
@@ -89,19 +88,63 @@ pub(crate) fn truncate_content(content: &mut String, limit: usize) {
content.push_str(&suffix_template.replace("%BYTES%", &dropped.to_string()));
}
#[derive(Clone, PartialEq, Eq)]
pub struct ImageAttachment {
mime_type: String,
data: Arc<[u8]>,
}
impl ImageAttachment {
pub fn new(mime_type: impl Into<String>, data: impl Into<Arc<[u8]>>) -> Self {
Self {
mime_type: mime_type.into(),
data: data.into(),
}
}
pub fn mime_type(&self) -> &str {
&self.mime_type
}
pub fn data(&self) -> &[u8] {
&self.data
}
}
impl fmt::Debug for ImageAttachment {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ImageAttachment")
.field("mime_type", &self.mime_type)
.field("bytes", &self.data.len())
.finish()
}
}
/// Request-local binary payload emitted by a tool.
///
/// Attachments are deliberately excluded from serde. They may be projected into
/// the immediately following provider request, but never into persisted history,
/// protocol events, logs, or telemetry.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Attachment {
Image(ImageAttachment),
}
/// Tool execution result.
///
/// Every output has a mandatory `summary` (1-2 lines) that persists in
/// conversation history even after pruning. The optional `content` carries
/// full details and is removed by the Prune mechanism when the context
/// grows too large.
/// full text details. `attachments` are request-local and are never serialized.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolOutput {
/// Short summary (1-2 lines). Always remains in history.
pub summary: String,
/// Detailed output. Removed by Prune when old enough.
/// Detailed text output. Removed by Prune when old enough.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
/// Structured binary payloads for the immediately following model request.
#[serde(skip, default)]
pub attachments: Vec<Attachment>,
}
impl From<String> for ToolOutput {
@@ -110,6 +153,7 @@ impl From<String> for ToolOutput {
ToolOutput {
summary: s,
content: None,
attachments: Vec::new(),
}
} else {
let lines = s.lines().count();
@@ -118,6 +162,7 @@ impl From<String> for ToolOutput {
ToolOutput {
summary,
content: Some(s),
attachments: Vec::new(),
}
}
}
@@ -364,6 +409,9 @@ pub struct ToolResult {
/// Whether this is an error
#[serde(default)]
pub is_error: bool,
/// Request-local structured payloads. Never serialized or persisted.
#[serde(skip, default)]
pub attachments: Vec<Attachment>,
}
impl ToolResult {
@@ -374,6 +422,7 @@ impl ToolResult {
summary: output.summary,
content: output.content,
is_error: false,
attachments: output.attachments,
}
}
@@ -384,6 +433,7 @@ impl ToolResult {
summary: message.into(),
content: None,
is_error: true,
attachments: Vec::new(),
}
}
}