Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: make image tool results durably prunable

Browse Source
This commit is contained in:
Hare
2026-08-11 02:44:37 +09:00
parent 38b8f26a50
commit 8d2b8b690f
15 changed files with 469 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/llm-engine/src/llm_client/scheme/anthropic/request.rs
-3
View File
@@ -268,9 +268,6 @@ impl AnthropicScheme {
.iter()
.map(|p| match p {
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
ContentPart::Image { .. } => {
AnthropicContentPart::text(p.as_text().to_string())
}
ContentPart::Refusal { refusal } => {
AnthropicContentPart::text(refusal.clone())
}
crates/llm-engine/src/llm_client/scheme/openai_chat/request.rs
+71 -55
View File
@@ -5,10 +5,13 @@
use serde::Serialize;
use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
use crate::{
llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
},
tool::Attachment,
};
use super::OpenAIScheme;
@@ -185,6 +188,21 @@ impl OpenAIScheme {
/// - Assistant messages have role "assistant"
/// - Tool calls are within assistant messages as tool_calls array
/// - Tool results have role "tool" with tool_call_id
fn flush_pending_tool_result_images(
messages: &mut Vec<OpenAIMessage>,
pending_images: &mut Vec<OpenAIContentPart>,
) {
if !pending_images.is_empty() {
messages.push(OpenAIMessage {
role: "user".to_string(),
content: Some(OpenAIContent::Parts(std::mem::take(pending_images))),
tool_calls: vec![],
tool_call_id: None,
name: None,
});
}
}
fn convert_items_to_messages(
&self,
items: &[Item],
@@ -193,8 +211,15 @@ impl OpenAIScheme {
let mut messages = Vec::new();
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
let mut pending_assistant_text: Option<String> = None;
let mut pending_tool_result_images: Vec<OpenAIContentPart> = Vec::new();
for item in items {
if !matches!(item, Item::ToolResult { .. }) {
Self::flush_pending_tool_result_images(
&mut messages,
&mut pending_tool_result_images,
);
}
match item {
Item::Message { role, content, .. } => {
// Flush pending tool calls
@@ -209,41 +234,13 @@ impl OpenAIScheme {
Role::Assistant => "assistant",
Role::System => "system",
};
let has_image = matches!(role, Role::User)
&& supports_images
&& content
let message_content = OpenAIContent::Text(
content
.iter()
.any(|part| matches!(part, ContentPart::Image { .. }));
let message_content = if has_image {
OpenAIContent::Parts(
content
.iter()
.map(|part| match part {
ContentPart::Text { text } => {
OpenAIContentPart::Text { text: text.clone() }
}
ContentPart::Image { media_type, source } => {
OpenAIContentPart::ImageUrl {
image_url: ImageUrl {
url: image_data_url(media_type, source.data()),
},
}
}
ContentPart::Refusal { refusal } => OpenAIContentPart::Text {
text: refusal.clone(),
},
})
.collect(),
)
} else {
OpenAIContent::Text(
content
.iter()
.map(ContentPart::as_text)
.collect::<Vec<_>>()
.join(""),
)
};
.map(ContentPart::as_text)
.collect::<Vec<_>>()
.join(""),
);
messages.push(OpenAIMessage {
role: openai_role.to_string(),
@@ -277,19 +274,35 @@ impl OpenAIScheme {
call_id,
summary,
content,
attachments,
..
} => {
// Flush pending tool calls before tool result
// OpenAI requires every parallel tool result before a new user message.
self.flush_pending_assistant(
&mut messages,
&mut pending_tool_calls,
&mut pending_assistant_text,
);
let text = match content {
let mut text = match content {
Some(c) => format!("{summary}\n{c}"),
None => summary.clone(),
};
if supports_images {
pending_tool_result_images.extend(attachments.iter().map(|attachment| {
let Attachment::Image(image) = attachment;
OpenAIContentPart::ImageUrl {
image_url: ImageUrl {
url: image_data_url(image.mime_type(), image.data()),
},
}
}));
} else if !attachments.is_empty() {
text.push_str(&format!(
"\n[{} image attachment(s) omitted: model does not support images]",
attachments.len()
));
}
messages.push(OpenAIMessage {
role: "tool".to_string(),
content: Some(OpenAIContent::Text(text)),
@@ -317,6 +330,7 @@ impl OpenAIScheme {
&mut pending_tool_calls,
&mut pending_assistant_text,
);
Self::flush_pending_tool_result_images(&mut messages, &mut pending_tool_result_images);
messages
}
@@ -481,28 +495,27 @@ mod tests {
}
#[test]
fn parallel_tool_results_precede_synthetic_image_message() {
fn parallel_tool_results_precede_durable_image_projection() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let request = Request::new()
.item(Item::tool_call("call_image", "ViewImage", "{}"))
.item(Item::tool_call("call_text", "Read", "{}"))
.item(Item::tool_result_item(
.item(Item::tool_result_item_with_attachments(
"call_image",
"Attached image",
None,
false,
vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", image),
)],
))
.item(Item::tool_result_item(
"call_text",
"Read text",
None,
false,
))
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
));
let json = serde_json::to_value(
&scheme
.build_request("gpt-4o", &request, &vision_cap())
@@ -518,7 +531,7 @@ mod tests {
}
#[test]
fn tool_image_is_structured_as_following_user_content_without_persisting_bytes() {
fn durable_tool_image_is_deterministically_lowered_to_following_user_content() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
@@ -533,8 +546,8 @@ mod tests {
vec![attachment],
);
let persisted = serde_json::to_string(&item).unwrap();
assert!(!persisted.contains("base64"));
assert!(!persisted.contains("attachments"));
assert!(persisted.contains("attachments"));
let restored: Item = serde_json::from_str(&persisted).unwrap();
let request = Request::new()
.item(Item::tool_call(
@@ -542,13 +555,16 @@ mod tests {
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
.item(restored);
let body = scheme.build_request("gpt-4o", &request, &vision_cap());
let json = serde_json::to_value(&body.messages).unwrap();
let rebuilt = serde_json::to_value(
&scheme
.build_request("gpt-4o", &request, &vision_cap())
.messages,
)
.unwrap();
assert_eq!(rebuilt, json);
assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool");
crates/llm-engine/src/llm_client/scheme/openai_responses/request.rs
+57 -29
View File
@@ -7,14 +7,31 @@
use serde::{Serialize, Serializer};
use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
use crate::{
llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
},
tool::Attachment,
};
use super::OpenAIResponsesScheme;
#[derive(Debug, Serialize)]
#[serde(untagged)]
pub(crate) enum FunctionCallOutputBody {
Text(String),
ContentItems(Vec<FunctionCallOutputContentItem>),
}
#[derive(Debug, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub(crate) enum FunctionCallOutputContentItem {
InputText { text: String },
InputImage { image_url: String },
}
/// `/v1/responses` のリクエスト body。
#[derive(Debug, Serialize)]
pub(crate) struct ResponsesRequest {
@@ -89,7 +106,10 @@ pub(crate) enum InputItem {
arguments: String,
},
/// function tool の結果（user 側）。
FunctionCallOutput { call_id: String, output: String },
FunctionCallOutput {
call_id: String,
output: FunctionCallOutputBody,
},
/// reasoning item。`encrypted_content` があれば必ず添える。
Reasoning {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -114,10 +134,6 @@ pub(crate) enum InputContent {
/// user / developer 側のテキスト
InputText { text: String },
/// user 側の画像
InputImage {
image_url: String,
detail: &'static str,
},
/// assistant 側のテキスト
OutputText { text: String },
}
@@ -249,15 +265,6 @@ fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputIte
.iter()
.map(|part| match part {
ContentPart::Text { text } => text_variant(text.clone()),
ContentPart::Image { media_type, source }
if matches!(role, Role::User) && supports_images =>
{
InputContent::InputImage {
image_url: image_data_url(media_type, source.data()),
detail: "auto",
}
}
ContentPart::Image { .. } => text_variant(part.as_text().to_string()),
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
})
.collect();
@@ -284,12 +291,30 @@ fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputIte
call_id,
summary,
content,
attachments,
..
} => {
let output = match content {
let text = match content {
Some(c) => format!("{summary}\n{c}"),
None => summary.clone(),
};
let output = if attachments.is_empty() {
FunctionCallOutputBody::Text(text)
} else if supports_images {
let mut parts = vec![FunctionCallOutputContentItem::InputText { text }];
parts.extend(attachments.iter().map(|attachment| {
let Attachment::Image(image) = attachment;
FunctionCallOutputContentItem::InputImage {
image_url: image_data_url(image.mime_type(), image.data()),
}
}));
FunctionCallOutputBody::ContentItems(parts)
} else {
FunctionCallOutputBody::Text(format!(
"{text}\n[{} image attachment(s) omitted: model does not support images]",
attachments.len()
))
};
out.push(InputItem::FunctionCallOutput {
call_id: call_id.clone(),
output,
@@ -701,7 +726,7 @@ mod tests {
}
#[test]
fn synthetic_user_image_uses_responses_input_image_content() {
fn durable_tool_image_uses_function_call_output_content_items() {
let scheme = OpenAIResponsesScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let item = Item::tool_result_item_with_attachments(
@@ -710,32 +735,35 @@ mod tests {
None,
false,
vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", image.clone()),
crate::tool::ImageAttachment::new("image/png", image),
)],
);
let persisted = serde_json::to_string(&item).unwrap();
let restored: Item = serde_json::from_str(&persisted).unwrap();
let req = Request::new()
.item(Item::tool_call(
"call_image",
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
.item(restored);
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
let json = serde_json::to_value(&body).unwrap();
assert_eq!(json["input"][1]["type"], "function_call_output");
assert_eq!(json["input"][2]["type"], "message");
assert_eq!(json["input"][2]["content"][0]["type"], "input_image");
assert_eq!(json["input"].as_array().unwrap().len(), 2);
assert_eq!(json["input"][1]["output"][0]["type"], "input_text");
assert_eq!(json["input"][1]["output"][1]["type"], "input_image");
assert!(
json["input"][2]["content"][0]["image_url"]
json["input"][1]["output"][1]["image_url"]
.as_str()
.unwrap()
.starts_with("data:image/png;base64,")
);
let rebuilt =
serde_json::to_value(scheme.build_request("gpt-5", &req, &cap_with_reasoning()))
.unwrap();
assert_eq!(rebuilt["input"], json["input"]);
let mut no_vision = cap_with_reasoning();
no_vision.vision = false;
crates/llm-engine/src/llm_client/types.rs
+4 -59
View File
@@ -124,8 +124,8 @@ pub enum Item {
/// Whether the tool result represents an execution error.
#[serde(default, skip_serializing_if = "is_false")]
is_error: bool,
/// Request-local structured payloads. Never serialized or persisted.
#[serde(skip, default)]
/// Durable binary details (removed with `content` by normal pruning).
#[serde(default, skip_serializing_if = "Vec::is_empty")]
attachments: Vec<Attachment>,
},
@@ -264,7 +264,7 @@ impl Item {
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
}
/// Create a tool result item with request-local structured attachments.
/// Create a tool result item with durable, prunable structured attachments.
pub fn tool_result_item_with_attachments(
call_id: impl Into<String>,
summary: impl Into<String>,
@@ -282,13 +282,6 @@ impl Item {
}
}
/// Drop request-local attachments after constructing the provider request.
pub fn clear_transient_attachments(&mut self) {
if let Self::ToolResult { attachments, .. } = self {
attachments.clear();
}
}
/// Create a tool result item with summary and content.
pub fn tool_result_with_content(
call_id: impl Into<String>,
@@ -457,38 +450,6 @@ pub fn parse_tool_arguments(arguments: &str) -> serde_json::Value {
// Content Parts - Components within message items
// ============================================================================
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ImageSource {
bytes: usize,
#[serde(skip, default)]
data: Arc<[u8]>,
}
impl ImageSource {
pub fn new(data: Arc<[u8]>) -> Self {
Self {
bytes: data.len(),
data,
}
}
pub fn data(&self) -> &[u8] {
&self.data
}
pub fn bytes(&self) -> usize {
self.bytes
}
}
impl fmt::Debug for ImageSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ImageSource")
.field("bytes", &self.bytes)
.finish()
}
}
/// Content part within a message item
///
/// Text content is role-agnostic; the containing Item's Role determines
@@ -502,12 +463,6 @@ pub enum ContentPart {
text: String,
},
/// Request-local image content. The source bytes are never serialized.
Image {
media_type: String,
source: ImageSource,
},
/// Refusal content (for assistant messages)
Refusal {
/// The refusal message
@@ -528,20 +483,10 @@ impl ContentPart {
}
}
pub fn image(media_type: impl Into<String>, data: Arc<[u8]>) -> Self {
Self::Image {
media_type: media_type.into(),
source: ImageSource::new(data),
}
}
/// Get a bounded textual projection. Image content is represented by an
/// explicit placeholder rather than silently becoming an empty string,
/// filesystem path, data URL, or base64 body.
/// Get a textual projection of the content part.
pub fn as_text(&self) -> &str {
match self {
Self::Text { text } => text,
Self::Image { .. } => "[image attachment omitted]",
Self::Refusal { refusal } => refusal,
}
}