fix: make image tool results durably prunable

This commit is contained in:
2026-08-11 02:44:37 +09:00
parent 38b8f26a50
commit 8d2b8b690f
15 changed files with 469 additions and 268 deletions
Generated
+1
View File
@@ -3895,6 +3895,7 @@ name = "session-store"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64 0.22.1",
"futures", "futures",
"llm-engine", "llm-engine",
"protocol", "protocol",
+21 -72
View File
@@ -19,19 +19,15 @@ use crate::{
}, },
llm_client::{ llm_client::{
ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream, ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream,
ToolDefinition, ToolDefinition, error::is_retryable, event::Event, retry::RetryPolicy,
error::is_retryable, transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT, types::parse_tool_arguments,
event::Event,
retry::RetryPolicy,
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT,
types::{ContentPart, parse_tool_arguments},
}, },
state::{EngineState, Locked, Mutable}, state::{EngineState, Locked, Mutable},
timeline::event::{ErrorEvent, StatusEvent, UsageEvent}, timeline::event::{ErrorEvent, StatusEvent, UsageEvent},
timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector}, timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector},
tool::{ tool::{
Attachment, ToolCall, ToolDefinition as EngineToolDefinition, ToolError, ToolCall, ToolDefinition as EngineToolDefinition, ToolError, ToolExecutionContext,
ToolExecutionContext, ToolOutputLimits, ToolResult, truncate_content, ToolOutputLimits, ToolResult, truncate_content,
}, },
tool_server::{ToolServer, ToolServerHandle}, tool_server::{ToolServer, ToolServerHandle},
}; };
@@ -159,33 +155,6 @@ enum StreamCompletion {
Interrupted { reason: String }, Interrupted { reason: String },
} }
fn project_transient_attachments(items: &mut Vec<Item>) {
let mut projected = Vec::with_capacity(items.len());
let mut pending_parts = Vec::new();
for mut item in items.drain(..) {
let is_tool_result = matches!(item, Item::ToolResult { .. });
if !is_tool_result && !pending_parts.is_empty() {
projected.push(Item::user_message_parts(std::mem::take(&mut pending_parts)));
}
if let Item::ToolResult { attachments, .. } = &item {
for attachment in attachments {
let Attachment::Image(image) = attachment;
pending_parts.push(ContentPart::image(
image.mime_type(),
Arc::from(image.data()),
));
}
}
item.clear_transient_attachments();
projected.push(item);
}
if !pending_parts.is_empty() {
projected.push(Item::user_message_parts(pending_parts));
}
*items = projected;
}
pub struct Engine<C: LlmClient, S: EngineState = Mutable> { pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
/// LLM client /// LLM client
client: C, client: C,
@@ -1280,8 +1249,6 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
cb(current_llm_call); cb(current_llm_call);
} }
project_transient_attachments(&mut request_context);
// Stream LLM response // Stream LLM response
self.emit_lifecycle_trace( self.emit_lifecycle_trace(
current_turn, current_turn,
@@ -1290,12 +1257,6 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
items_trace_payload(&request_context, tool_definitions.len(), None, false), items_trace_payload(&request_context, tool_definitions.len(), None, false),
); );
let request = self.build_request(&tool_definitions, &request_context); let request = self.build_request(&tool_definitions, &request_context);
// Structured attachments are single-use provider payloads. `request`
// owns its Arc-backed copy; history is cleared before any later turn
// can persist or resend the binary body.
for item in &mut self.history {
item.clear_transient_attachments();
}
self.emit_lifecycle_trace( self.emit_lifecycle_trace(
current_turn, current_turn,
current_llm_call, current_llm_call,
@@ -2199,48 +2160,36 @@ fn item_kind(item: &Item) -> &'static str {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::llm_client::types::Role; use crate::tool::{Attachment, ImageAttachment};
use std::time::Duration; use std::time::Duration;
#[test] #[test]
fn transient_tool_attachment_becomes_non_persistent_user_image_part() { fn tool_attachment_round_trips_through_durable_history_json() {
let body: Arc<[u8]> = Arc::from(&b"secret-image-body"[..]); let body: Arc<[u8]> = Arc::from(&b"image-body"[..]);
let mut items = vec![Item::tool_result_item_with_attachments( let items = vec![Item::tool_result_item_with_attachments(
"call_image", "call_image",
"attached", "attached",
None, None,
false, false,
vec![Attachment::Image(crate::tool::ImageAttachment::new( vec![Attachment::Image(ImageAttachment::new(
"image/png", "image/png",
body.clone(), body.clone(),
))], ))],
)]; )];
project_transient_attachments(&mut items);
assert!(matches!(
&items[0],
Item::ToolResult { attachments, .. } if attachments.is_empty()
));
match &items[1] {
Item::Message {
role: Role::User,
content,
..
} => match &content[0] {
ContentPart::Image { media_type, source } => {
assert_eq!(media_type, "image/png");
assert_eq!(source.data(), body.as_ref());
assert_eq!(source.bytes(), body.len());
assert_eq!(content[0].as_text(), "[image attachment omitted]");
}
other => panic!("unexpected content part: {other:?}"),
},
other => panic!("unexpected projected item: {other:?}"),
}
let persisted = serde_json::to_string(&items).unwrap(); let persisted = serde_json::to_string(&items).unwrap();
assert!(!persisted.contains("secret-image-body")); assert!(persisted.contains("aW1hZ2UtYm9keQ=="));
assert!(!persisted.contains("base64")); let restored: Vec<Item> = serde_json::from_str(&persisted).unwrap();
assert_eq!(restored, items);
assert!(matches!(
&restored[0],
Item::ToolResult { attachments, .. }
if matches!(
attachments.as_slice(),
[Attachment::Image(image)]
if image.mime_type() == "image/png" && image.data() == body.as_ref()
)
));
} }
#[tokio::test] #[tokio::test]
@@ -268,9 +268,6 @@ impl AnthropicScheme {
.iter() .iter()
.map(|p| match p { .map(|p| match p {
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()), ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
ContentPart::Image { .. } => {
AnthropicContentPart::text(p.as_text().to_string())
}
ContentPart::Refusal { refusal } => { ContentPart::Refusal { refusal } => {
AnthropicContentPart::text(refusal.clone()) AnthropicContentPart::text(refusal.clone())
} }
@@ -5,10 +5,13 @@
use serde::Serialize; use serde::Serialize;
use serde_json::Value; use serde_json::Value;
use crate::llm_client::{ use crate::{
llm_client::{
Request, Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport}, capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments}, types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
},
tool::Attachment,
}; };
use super::OpenAIScheme; use super::OpenAIScheme;
@@ -185,6 +188,21 @@ impl OpenAIScheme {
/// - Assistant messages have role "assistant" /// - Assistant messages have role "assistant"
/// - Tool calls are within assistant messages as tool_calls array /// - Tool calls are within assistant messages as tool_calls array
/// - Tool results have role "tool" with tool_call_id /// - Tool results have role "tool" with tool_call_id
fn flush_pending_tool_result_images(
messages: &mut Vec<OpenAIMessage>,
pending_images: &mut Vec<OpenAIContentPart>,
) {
if !pending_images.is_empty() {
messages.push(OpenAIMessage {
role: "user".to_string(),
content: Some(OpenAIContent::Parts(std::mem::take(pending_images))),
tool_calls: vec![],
tool_call_id: None,
name: None,
});
}
}
fn convert_items_to_messages( fn convert_items_to_messages(
&self, &self,
items: &[Item], items: &[Item],
@@ -193,8 +211,15 @@ impl OpenAIScheme {
let mut messages = Vec::new(); let mut messages = Vec::new();
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new(); let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
let mut pending_assistant_text: Option<String> = None; let mut pending_assistant_text: Option<String> = None;
let mut pending_tool_result_images: Vec<OpenAIContentPart> = Vec::new();
for item in items { for item in items {
if !matches!(item, Item::ToolResult { .. }) {
Self::flush_pending_tool_result_images(
&mut messages,
&mut pending_tool_result_images,
);
}
match item { match item {
Item::Message { role, content, .. } => { Item::Message { role, content, .. } => {
// Flush pending tool calls // Flush pending tool calls
@@ -209,41 +234,13 @@ impl OpenAIScheme {
Role::Assistant => "assistant", Role::Assistant => "assistant",
Role::System => "system", Role::System => "system",
}; };
let has_image = matches!(role, Role::User) let message_content = OpenAIContent::Text(
&& supports_images
&& content
.iter()
.any(|part| matches!(part, ContentPart::Image { .. }));
let message_content = if has_image {
OpenAIContent::Parts(
content
.iter()
.map(|part| match part {
ContentPart::Text { text } => {
OpenAIContentPart::Text { text: text.clone() }
}
ContentPart::Image { media_type, source } => {
OpenAIContentPart::ImageUrl {
image_url: ImageUrl {
url: image_data_url(media_type, source.data()),
},
}
}
ContentPart::Refusal { refusal } => OpenAIContentPart::Text {
text: refusal.clone(),
},
})
.collect(),
)
} else {
OpenAIContent::Text(
content content
.iter() .iter()
.map(ContentPart::as_text) .map(ContentPart::as_text)
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join(""), .join(""),
) );
};
messages.push(OpenAIMessage { messages.push(OpenAIMessage {
role: openai_role.to_string(), role: openai_role.to_string(),
@@ -277,19 +274,35 @@ impl OpenAIScheme {
call_id, call_id,
summary, summary,
content, content,
attachments,
.. ..
} => { } => {
// Flush pending tool calls before tool result // OpenAI requires every parallel tool result before a new user message.
self.flush_pending_assistant( self.flush_pending_assistant(
&mut messages, &mut messages,
&mut pending_tool_calls, &mut pending_tool_calls,
&mut pending_assistant_text, &mut pending_assistant_text,
); );
let text = match content { let mut text = match content {
Some(c) => format!("{summary}\n{c}"), Some(c) => format!("{summary}\n{c}"),
None => summary.clone(), None => summary.clone(),
}; };
if supports_images {
pending_tool_result_images.extend(attachments.iter().map(|attachment| {
let Attachment::Image(image) = attachment;
OpenAIContentPart::ImageUrl {
image_url: ImageUrl {
url: image_data_url(image.mime_type(), image.data()),
},
}
}));
} else if !attachments.is_empty() {
text.push_str(&format!(
"\n[{} image attachment(s) omitted: model does not support images]",
attachments.len()
));
}
messages.push(OpenAIMessage { messages.push(OpenAIMessage {
role: "tool".to_string(), role: "tool".to_string(),
content: Some(OpenAIContent::Text(text)), content: Some(OpenAIContent::Text(text)),
@@ -317,6 +330,7 @@ impl OpenAIScheme {
&mut pending_tool_calls, &mut pending_tool_calls,
&mut pending_assistant_text, &mut pending_assistant_text,
); );
Self::flush_pending_tool_result_images(&mut messages, &mut pending_tool_result_images);
messages messages
} }
@@ -481,28 +495,27 @@ mod tests {
} }
#[test] #[test]
fn parallel_tool_results_precede_synthetic_image_message() { fn parallel_tool_results_precede_durable_image_projection() {
let scheme = OpenAIScheme::new(); let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]); let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let request = Request::new() let request = Request::new()
.item(Item::tool_call("call_image", "ViewImage", "{}")) .item(Item::tool_call("call_image", "ViewImage", "{}"))
.item(Item::tool_call("call_text", "Read", "{}")) .item(Item::tool_call("call_text", "Read", "{}"))
.item(Item::tool_result_item( .item(Item::tool_result_item_with_attachments(
"call_image", "call_image",
"Attached image", "Attached image",
None, None,
false, false,
vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", image),
)],
)) ))
.item(Item::tool_result_item( .item(Item::tool_result_item(
"call_text", "call_text",
"Read text", "Read text",
None, None,
false, false,
)) ));
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let json = serde_json::to_value( let json = serde_json::to_value(
&scheme &scheme
.build_request("gpt-4o", &request, &vision_cap()) .build_request("gpt-4o", &request, &vision_cap())
@@ -518,7 +531,7 @@ mod tests {
} }
#[test] #[test]
fn tool_image_is_structured_as_following_user_content_without_persisting_bytes() { fn durable_tool_image_is_deterministically_lowered_to_following_user_content() {
let scheme = OpenAIScheme::new(); let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]); let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new( let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
@@ -533,8 +546,8 @@ mod tests {
vec![attachment], vec![attachment],
); );
let persisted = serde_json::to_string(&item).unwrap(); let persisted = serde_json::to_string(&item).unwrap();
assert!(!persisted.contains("base64")); assert!(persisted.contains("attachments"));
assert!(!persisted.contains("attachments")); let restored: Item = serde_json::from_str(&persisted).unwrap();
let request = Request::new() let request = Request::new()
.item(Item::tool_call( .item(Item::tool_call(
@@ -542,13 +555,16 @@ mod tests {
"ViewImage", "ViewImage",
r#"{"path":"a.png"}"#, r#"{"path":"a.png"}"#,
)) ))
.item(item) .item(restored);
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-4o", &request, &vision_cap()); let body = scheme.build_request("gpt-4o", &request, &vision_cap());
let json = serde_json::to_value(&body.messages).unwrap(); let json = serde_json::to_value(&body.messages).unwrap();
let rebuilt = serde_json::to_value(
&scheme
.build_request("gpt-4o", &request, &vision_cap())
.messages,
)
.unwrap();
assert_eq!(rebuilt, json);
assert_eq!(json[0]["role"], "assistant"); assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool"); assert_eq!(json[1]["role"], "tool");
@@ -7,14 +7,31 @@
use serde::{Serialize, Serializer}; use serde::{Serialize, Serializer};
use serde_json::Value; use serde_json::Value;
use crate::llm_client::{ use crate::{
llm_client::{
Request, Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport}, capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments}, types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
},
tool::Attachment,
}; };
use super::OpenAIResponsesScheme; use super::OpenAIResponsesScheme;
#[derive(Debug, Serialize)]
#[serde(untagged)]
pub(crate) enum FunctionCallOutputBody {
Text(String),
ContentItems(Vec<FunctionCallOutputContentItem>),
}
#[derive(Debug, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub(crate) enum FunctionCallOutputContentItem {
InputText { text: String },
InputImage { image_url: String },
}
/// `/v1/responses` のリクエスト body。 /// `/v1/responses` のリクエスト body。
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
pub(crate) struct ResponsesRequest { pub(crate) struct ResponsesRequest {
@@ -89,7 +106,10 @@ pub(crate) enum InputItem {
arguments: String, arguments: String,
}, },
/// function tool の結果(user 側)。 /// function tool の結果(user 側)。
FunctionCallOutput { call_id: String, output: String }, FunctionCallOutput {
call_id: String,
output: FunctionCallOutputBody,
},
/// reasoning item。`encrypted_content` があれば必ず添える。 /// reasoning item。`encrypted_content` があれば必ず添える。
Reasoning { Reasoning {
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
@@ -114,10 +134,6 @@ pub(crate) enum InputContent {
/// user / developer 側のテキスト /// user / developer 側のテキスト
InputText { text: String }, InputText { text: String },
/// user 側の画像 /// user 側の画像
InputImage {
image_url: String,
detail: &'static str,
},
/// assistant 側のテキスト /// assistant 側のテキスト
OutputText { text: String }, OutputText { text: String },
} }
@@ -249,15 +265,6 @@ fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputIte
.iter() .iter()
.map(|part| match part { .map(|part| match part {
ContentPart::Text { text } => text_variant(text.clone()), ContentPart::Text { text } => text_variant(text.clone()),
ContentPart::Image { media_type, source }
if matches!(role, Role::User) && supports_images =>
{
InputContent::InputImage {
image_url: image_data_url(media_type, source.data()),
detail: "auto",
}
}
ContentPart::Image { .. } => text_variant(part.as_text().to_string()),
ContentPart::Refusal { refusal } => text_variant(refusal.clone()), ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
}) })
.collect(); .collect();
@@ -284,12 +291,30 @@ fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputIte
call_id, call_id,
summary, summary,
content, content,
attachments,
.. ..
} => { } => {
let output = match content { let text = match content {
Some(c) => format!("{summary}\n{c}"), Some(c) => format!("{summary}\n{c}"),
None => summary.clone(), None => summary.clone(),
}; };
let output = if attachments.is_empty() {
FunctionCallOutputBody::Text(text)
} else if supports_images {
let mut parts = vec![FunctionCallOutputContentItem::InputText { text }];
parts.extend(attachments.iter().map(|attachment| {
let Attachment::Image(image) = attachment;
FunctionCallOutputContentItem::InputImage {
image_url: image_data_url(image.mime_type(), image.data()),
}
}));
FunctionCallOutputBody::ContentItems(parts)
} else {
FunctionCallOutputBody::Text(format!(
"{text}\n[{} image attachment(s) omitted: model does not support images]",
attachments.len()
))
};
out.push(InputItem::FunctionCallOutput { out.push(InputItem::FunctionCallOutput {
call_id: call_id.clone(), call_id: call_id.clone(),
output, output,
@@ -701,7 +726,7 @@ mod tests {
} }
#[test] #[test]
fn synthetic_user_image_uses_responses_input_image_content() { fn durable_tool_image_uses_function_call_output_content_items() {
let scheme = OpenAIResponsesScheme::new(); let scheme = OpenAIResponsesScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]); let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let item = Item::tool_result_item_with_attachments( let item = Item::tool_result_item_with_attachments(
@@ -710,32 +735,35 @@ mod tests {
None, None,
false, false,
vec![crate::tool::Attachment::Image( vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", image.clone()), crate::tool::ImageAttachment::new("image/png", image),
)], )],
); );
let persisted = serde_json::to_string(&item).unwrap();
let restored: Item = serde_json::from_str(&persisted).unwrap();
let req = Request::new() let req = Request::new()
.item(Item::tool_call( .item(Item::tool_call(
"call_image", "call_image",
"ViewImage", "ViewImage",
r#"{"path":"a.png"}"#, r#"{"path":"a.png"}"#,
)) ))
.item(item) .item(restored);
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning()); let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
let json = serde_json::to_value(&body).unwrap(); let json = serde_json::to_value(&body).unwrap();
assert_eq!(json["input"][1]["type"], "function_call_output"); assert_eq!(json["input"][1]["type"], "function_call_output");
assert_eq!(json["input"][2]["type"], "message"); assert_eq!(json["input"].as_array().unwrap().len(), 2);
assert_eq!(json["input"][2]["content"][0]["type"], "input_image"); assert_eq!(json["input"][1]["output"][0]["type"], "input_text");
assert_eq!(json["input"][1]["output"][1]["type"], "input_image");
assert!( assert!(
json["input"][2]["content"][0]["image_url"] json["input"][1]["output"][1]["image_url"]
.as_str() .as_str()
.unwrap() .unwrap()
.starts_with("data:image/png;base64,") .starts_with("data:image/png;base64,")
); );
let rebuilt =
serde_json::to_value(scheme.build_request("gpt-5", &req, &cap_with_reasoning()))
.unwrap();
assert_eq!(rebuilt["input"], json["input"]);
let mut no_vision = cap_with_reasoning(); let mut no_vision = cap_with_reasoning();
no_vision.vision = false; no_vision.vision = false;
+4 -59
View File
@@ -124,8 +124,8 @@ pub enum Item {
/// Whether the tool result represents an execution error. /// Whether the tool result represents an execution error.
#[serde(default, skip_serializing_if = "is_false")] #[serde(default, skip_serializing_if = "is_false")]
is_error: bool, is_error: bool,
/// Request-local structured payloads. Never serialized or persisted. /// Durable binary details (removed with `content` by normal pruning).
#[serde(skip, default)] #[serde(default, skip_serializing_if = "Vec::is_empty")]
attachments: Vec<Attachment>, attachments: Vec<Attachment>,
}, },
@@ -264,7 +264,7 @@ impl Item {
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new()) Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
} }
/// Create a tool result item with request-local structured attachments. /// Create a tool result item with durable, prunable structured attachments.
pub fn tool_result_item_with_attachments( pub fn tool_result_item_with_attachments(
call_id: impl Into<String>, call_id: impl Into<String>,
summary: impl Into<String>, summary: impl Into<String>,
@@ -282,13 +282,6 @@ impl Item {
} }
} }
/// Drop request-local attachments after constructing the provider request.
pub fn clear_transient_attachments(&mut self) {
if let Self::ToolResult { attachments, .. } = self {
attachments.clear();
}
}
/// Create a tool result item with summary and content. /// Create a tool result item with summary and content.
pub fn tool_result_with_content( pub fn tool_result_with_content(
call_id: impl Into<String>, call_id: impl Into<String>,
@@ -457,38 +450,6 @@ pub fn parse_tool_arguments(arguments: &str) -> serde_json::Value {
// Content Parts - Components within message items // Content Parts - Components within message items
// ============================================================================ // ============================================================================
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ImageSource {
bytes: usize,
#[serde(skip, default)]
data: Arc<[u8]>,
}
impl ImageSource {
pub fn new(data: Arc<[u8]>) -> Self {
Self {
bytes: data.len(),
data,
}
}
pub fn data(&self) -> &[u8] {
&self.data
}
pub fn bytes(&self) -> usize {
self.bytes
}
}
impl fmt::Debug for ImageSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ImageSource")
.field("bytes", &self.bytes)
.finish()
}
}
/// Content part within a message item /// Content part within a message item
/// ///
/// Text content is role-agnostic; the containing Item's Role determines /// Text content is role-agnostic; the containing Item's Role determines
@@ -502,12 +463,6 @@ pub enum ContentPart {
text: String, text: String,
}, },
/// Request-local image content. The source bytes are never serialized.
Image {
media_type: String,
source: ImageSource,
},
/// Refusal content (for assistant messages) /// Refusal content (for assistant messages)
Refusal { Refusal {
/// The refusal message /// The refusal message
@@ -528,20 +483,10 @@ impl ContentPart {
} }
} }
pub fn image(media_type: impl Into<String>, data: Arc<[u8]>) -> Self { /// Get a textual projection of the content part.
Self::Image {
media_type: media_type.into(),
source: ImageSource::new(data),
}
}
/// Get a bounded textual projection. Image content is represented by an
/// explicit placeholder rather than silently becoming an empty string,
/// filesystem path, data URL, or base64 body.
pub fn as_text(&self) -> &str { pub fn as_text(&self) -> &str {
match self { match self {
Self::Text { text } => text, Self::Text { text } => text,
Self::Image { .. } => "[image attachment omitted]",
Self::Refusal { refusal } => refusal, Self::Refusal { refusal } => refusal,
} }
} }
+48 -9
View File
@@ -110,25 +110,30 @@ impl Default for PruneConfig {
} }
} }
/// Set `content = None` on each `Item::ToolResult` at the given indices. /// Remove detailed text and attachments from each `Item::ToolResult` at the given indices.
/// ///
/// Returns the number of items that were actually modified — items that /// The mandatory summary remains. Returns the number of items that were actually
/// are already content-less are counted as 0. Intended for use on a /// modified — results that already contain no detail are counted as 0. Intended
/// request-context clone (never on a persistent history). /// for use on a request-context clone (never on a persistent history).
pub fn project(items: &mut [Item], indices: &[usize]) -> usize { pub fn project(items: &mut [Item], indices: &[usize]) -> usize {
let mut count = 0; let mut count = 0;
for &i in indices { for &i in indices {
if let Item::ToolResult { content, .. } = &mut items[i] if let Item::ToolResult {
&& content.is_some() content,
attachments,
..
} = &mut items[i]
&& (content.is_some() || !attachments.is_empty())
{ {
*content = None; *content = None;
attachments.clear();
count += 1; count += 1;
} }
} }
count count
} }
/// Indices of `Item::ToolResult { content: Some(_), .. }` that lie before /// Indices of detailed `Item::ToolResult` values that lie before
/// the suffix protected by `protected_tokens`. Pure: does not mutate `items`. /// the suffix protected by `protected_tokens`. Pure: does not mutate `items`.
/// ///
/// Returns an empty vector when token estimates are unavailable (`NoData`) or /// Returns an empty vector when token estimates are unavailable (`NoData`) or
@@ -159,8 +164,10 @@ pub fn evaluate_candidates(
.enumerate() .enumerate()
.filter_map(|(i, item)| match item { .filter_map(|(i, item)| match item {
Item::ToolResult { Item::ToolResult {
content: Some(_), .. content,
} => Some(i), attachments,
..
} if content.is_some() || !attachments.is_empty() => Some(i),
_ => None, _ => None,
}) })
.collect(); .collect();
@@ -373,6 +380,38 @@ mod tests {
} }
} }
#[test]
fn project_drops_image_detail_but_keeps_summary_and_persistent_source() {
let original = vec![Item::tool_result_item_with_attachments(
"call_image",
"Attached image/png image (12 bytes)",
None,
false,
vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", b"image-body".to_vec()),
)],
)];
let mut request_context = original.clone();
let estimates = uniform_estimates(&original, 100);
assert_eq!(prunable_indices(&original, 0, &estimates), vec![0]);
assert_eq!(project(&mut request_context, &[0]), 1);
assert!(matches!(
&request_context[0],
Item::ToolResult {
summary,
content: None,
attachments,
..
} if summary == "Attached image/png image (12 bytes)" && attachments.is_empty()
));
assert!(matches!(
&original[0],
Item::ToolResult { attachments, .. } if attachments.len() == 1
));
}
#[test] #[test]
fn project_skips_already_pruned_items() { fn project_skips_already_pruned_items() {
// indices points at an item whose content is already None. // indices points at an item whose content is already None.
+41 -13
View File
@@ -6,7 +6,8 @@
use std::{collections::HashMap, fmt, sync::Arc}; use std::{collections::HashMap, fmt, sync::Arc};
use async_trait::async_trait; use async_trait::async_trait;
use serde::{Deserialize, Serialize}; use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
use serde_json::Value; use serde_json::Value;
use thiserror::Error; use thiserror::Error;
@@ -120,12 +121,39 @@ impl fmt::Debug for ImageAttachment {
} }
} }
/// Request-local binary payload emitted by a tool. #[derive(Serialize, Deserialize)]
/// struct ImageAttachmentWire {
/// Attachments are deliberately excluded from serde. They may be projected into mime_type: String,
/// the immediately following provider request, but never into persisted history, data: String,
/// protocol events, logs, or telemetry. }
#[derive(Debug, Clone, PartialEq, Eq)]
impl Serialize for ImageAttachment {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
ImageAttachmentWire {
mime_type: self.mime_type.clone(),
data: STANDARD.encode(self.data.as_ref()),
}
.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for ImageAttachment {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let wire = ImageAttachmentWire::deserialize(deserializer)?;
let data = STANDARD.decode(wire.data).map_err(D::Error::custom)?;
Ok(Self::new(wire.mime_type, data))
}
}
/// Durable binary detail emitted by a tool and handled by normal ToolResult pruning.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", content = "payload", rename_all = "snake_case")]
pub enum Attachment { pub enum Attachment {
Image(ImageAttachment), Image(ImageAttachment),
} }
@@ -133,8 +161,8 @@ pub enum Attachment {
/// Tool execution result. /// Tool execution result.
/// ///
/// Every output has a mandatory `summary` (1-2 lines) that persists in /// Every output has a mandatory `summary` (1-2 lines) that persists in
/// conversation history even after pruning. The optional `content` carries /// conversation history even after pruning. Optional text and binary details are
/// full text details. `attachments` are request-local and are never serialized. /// committed to history and may later be omitted only by normal ToolResult pruning.
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolOutput { pub struct ToolOutput {
/// Short summary (1-2 lines). Always remains in history. /// Short summary (1-2 lines). Always remains in history.
@@ -142,8 +170,8 @@ pub struct ToolOutput {
/// Detailed text output. Removed by Prune when old enough. /// Detailed text output. Removed by Prune when old enough.
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<String>, pub content: Option<String>,
/// Structured binary payloads for the immediately following model request. /// Durable binary details handled by the same pruning lifecycle as `content`.
#[serde(skip, default)] #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub attachments: Vec<Attachment>, pub attachments: Vec<Attachment>,
} }
@@ -409,8 +437,8 @@ pub struct ToolResult {
/// Whether this is an error /// Whether this is an error
#[serde(default)] #[serde(default)]
pub is_error: bool, pub is_error: bool,
/// Request-local structured payloads. Never serialized or persisted. /// Durable binary details (prunable with `content`).
#[serde(skip, default)] #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub attachments: Vec<Attachment>, pub attachments: Vec<Attachment>,
} }
+1
View File
@@ -6,6 +6,7 @@ edition.workspace = true
license.workspace = true license.workspace = true
[dependencies] [dependencies]
base64.workspace = true
llm-engine = { workspace = true } llm-engine = { workspace = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true } serde_json = { workspace = true }
+73 -11
View File
@@ -12,13 +12,36 @@
//! `Reasoning::encrypted_content` is preserved because OpenAI Responses ZDR //! `Reasoning::encrypted_content` is preserved because OpenAI Responses ZDR
//! requires it on stateless re-send. //! requires it on stateless re-send.
use llm_engine::llm_client::types::{ContentPart, Item, Role}; use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde::{Deserialize, Serialize}; use llm_engine::{
llm_client::types::{ContentPart, Item, Role},
tool::{Attachment, ImageAttachment},
};
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
fn is_false(value: &bool) -> bool { fn is_false(value: &bool) -> bool {
!*value !*value
} }
mod base64_bytes {
use super::*;
pub fn serialize<S>(data: &[u8], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&STANDARD.encode(data))
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
let encoded = String::deserialize(deserializer)?;
STANDARD.decode(encoded).map_err(D::Error::custom)
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")] #[serde(tag = "kind", rename_all = "snake_case")]
pub enum LoggedItem { pub enum LoggedItem {
@@ -36,6 +59,8 @@ pub enum LoggedItem {
summary: String, summary: String,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
content: Option<String>, content: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
attachments: Vec<LoggedAttachment>,
#[serde(default, skip_serializing_if = "is_false")] #[serde(default, skip_serializing_if = "is_false")]
is_error: bool, is_error: bool,
}, },
@@ -67,6 +92,16 @@ pub enum LoggedContentPart {
Refusal { refusal: String }, Refusal { refusal: String },
} }
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum LoggedAttachment {
Image {
mime_type: String,
#[serde(with = "base64_bytes")]
data: Vec<u8>,
},
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Item ↔ LoggedItem // Item ↔ LoggedItem
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -92,12 +127,14 @@ impl From<&Item> for LoggedItem {
call_id, call_id,
summary, summary,
content, content,
attachments,
is_error, is_error,
.. ..
} => Self::ToolResult { } => Self::ToolResult {
call_id: call_id.clone(), call_id: call_id.clone(),
summary: summary.clone(), summary: summary.clone(),
content: content.clone(), content: content.clone(),
attachments: attachments.iter().map(LoggedAttachment::from).collect(),
is_error: *is_error, is_error: *is_error,
}, },
Item::Reasoning { Item::Reasoning {
@@ -146,6 +183,7 @@ impl From<LoggedItem> for Item {
call_id, call_id,
summary, summary,
content, content,
attachments,
is_error, is_error,
} => Item::ToolResult { } => Item::ToolResult {
id: None, id: None,
@@ -153,7 +191,7 @@ impl From<LoggedItem> for Item {
summary, summary,
content, content,
is_error, is_error,
attachments: Vec::new(), attachments: attachments.into_iter().map(Attachment::from).collect(),
}, },
LoggedItem::Reasoning { LoggedItem::Reasoning {
text, text,
@@ -214,9 +252,6 @@ impl From<&ContentPart> for LoggedContentPart {
fn from(part: &ContentPart) -> Self { fn from(part: &ContentPart) -> Self {
match part { match part {
ContentPart::Text { text } => Self::Text { text: text.clone() }, ContentPart::Text { text } => Self::Text { text: text.clone() },
ContentPart::Image { .. } => Self::Text {
text: part.as_text().to_string(),
},
ContentPart::Refusal { refusal } => Self::Refusal { ContentPart::Refusal { refusal } => Self::Refusal {
refusal: refusal.clone(), refusal: refusal.clone(),
}, },
@@ -233,6 +268,27 @@ impl From<LoggedContentPart> for ContentPart {
} }
} }
impl From<&Attachment> for LoggedAttachment {
fn from(attachment: &Attachment) -> Self {
match attachment {
Attachment::Image(image) => Self::Image {
mime_type: image.mime_type().to_string(),
data: image.data().to_vec(),
},
}
}
}
impl From<LoggedAttachment> for Attachment {
fn from(attachment: LoggedAttachment) -> Self {
match attachment {
LoggedAttachment::Image { mime_type, data } => {
Self::Image(ImageAttachment::new(mime_type, data))
}
}
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -375,7 +431,7 @@ mod tests {
} }
#[test] #[test]
fn tool_result_persistence_drops_binary_attachments() { fn tool_result_persistence_round_trips_binary_attachments() {
let original = Item::tool_result_item_with_attachments( let original = Item::tool_result_item_with_attachments(
"call_image", "call_image",
"attached", "attached",
@@ -390,11 +446,17 @@ mod tests {
); );
let logged: LoggedItem = (&original).into(); let logged: LoggedItem = (&original).into();
let json = serde_json::to_string(&logged).unwrap(); let json = serde_json::to_string(&logged).unwrap();
assert!(!json.contains("secret-image-body")); assert!(json.contains("attachments"));
assert!(!json.contains("attachments")); assert!(json.contains("c2VjcmV0LWltYWdlLWJvZHk="));
match Item::from(logged) { let restored: LoggedItem = serde_json::from_str(&json).unwrap();
Item::ToolResult { attachments, .. } => assert!(attachments.is_empty()), match Item::from(restored) {
Item::ToolResult { attachments, .. } => assert!(matches!(
attachments.as_slice(),
[Attachment::Image(image)]
if image.mime_type() == "image/png"
&& image.data() == b"secret-image-body"
)),
other => panic!("unexpected variant: {other:?}"), other => panic!("unexpected variant: {other:?}"),
} }
} }
+30
View File
@@ -473,6 +473,36 @@ mod tests {
assert!(state.history[2].is_tool_result()); assert!(state.history[2].is_tool_result());
} }
#[test]
fn replay_restores_durable_tool_image_detail() {
let entry = LogEntry::ToolResult {
ts: 3500,
item: Item::tool_result_item_with_attachments(
"call_image",
"attached",
None,
false,
vec![llm_engine::tool::Attachment::Image(
llm_engine::tool::ImageAttachment::new("image/png", b"durable-image".to_vec()),
)],
)
.into(),
};
let persisted = serde_json::to_string(&entry).unwrap();
let restored_entry: LogEntry = serde_json::from_str(&persisted).unwrap();
let state = collect_state(&[restored_entry]);
assert!(matches!(
&state.history[0],
Item::ToolResult { attachments, .. }
if matches!(
attachments.as_slice(),
[llm_engine::tool::Attachment::Image(image)]
if image.data() == b"durable-image"
)
));
}
#[test] #[test]
fn replay_config_changed() { fn replay_config_changed() {
let state = collect_state(&[ let state = collect_state(&[
+5 -2
View File
@@ -101,7 +101,7 @@ fn meta_has_description_and_schema() {
} }
#[tokio::test] #[tokio::test]
async fn view_image_reads_scoped_bytes_without_serializing_them_as_text() { async fn view_image_reads_scoped_bytes_into_durable_tool_detail() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
let spill = TempDir::new().unwrap(); let spill = TempDir::new().unwrap();
let scope = scope_with_spill(dir.path(), spill.path()); let scope = scope_with_spill(dir.path(), spill.path());
@@ -119,7 +119,10 @@ async fn view_image_reads_scoped_bytes_without_serializing_them_as_text() {
assert_eq!(image.data(), png); assert_eq!(image.data(), png);
let serialized = serde_json::to_string(&output).unwrap(); let serialized = serde_json::to_string(&output).unwrap();
assert!(!serialized.contains("private-image-body")); assert!(!serialized.contains("private-image-body"));
assert!(!serialized.contains("attachments")); assert!(serialized.contains("attachments"));
let restored: llm_engine::tool::ToolOutput = serde_json::from_str(&serialized).unwrap();
let llm_engine::tool::Attachment::Image(restored_image) = &restored.attachments[0];
assert_eq!(restored_image.data(), png);
let escaped = call_err(&tool, json!({ "path": "../outside.png" })).await; let escaped = call_err(&tool, json!({ "path": "../outside.png" })).await;
assert!(escaped.to_string().contains("scope") || escaped.to_string().contains("path")); assert!(escaped.to_string().contains("scope") || escaped.to_string().contains("path"));
+62 -11
View File
@@ -271,9 +271,20 @@ impl SessionCapture {
}); });
} }
Item::ToolResult { Item::ToolResult {
summary, content, .. summary,
content,
attachments,
..
} => { } => {
let text = format!("{summary}\n{}", content.as_deref().unwrap_or_default()); let attachment_marker = if attachments.is_empty() {
String::new()
} else {
format!("\n[{} image attachment(s)]", attachments.len())
};
let text = format!(
"{summary}\n{}{attachment_marker}",
content.as_deref().unwrap_or_default(),
);
index.push(ReferenceEntry { index.push(ReferenceEntry {
id: SessionEntryRef::new(idx), id: SessionEntryRef::new(idx),
entry_range, entry_range,
@@ -506,21 +517,29 @@ fn render_item(
Item::ToolResult { Item::ToolResult {
summary, summary,
content, content,
attachments,
is_error, is_error,
.. ..
} => match detail { } => {
let attachment_line = if attachments.is_empty() {
String::new()
} else {
format!("\nattachments: {} image(s)", attachments.len())
};
match detail {
ReadDetail::Compact => format!( ReadDetail::Compact => format!(
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted)", "[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted){attachment_line}",
entry.id,
if *is_error { " error" } else { "" }
),
ReadDetail::Full => format!(
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}",
entry.id, entry.id,
if *is_error { " error" } else { "" }, if *is_error { " error" } else { "" },
content.as_deref().unwrap_or_default()
), ),
}, ReadDetail::Full => format!(
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}{attachment_line}",
entry.id,
if *is_error { " error" } else { "" },
content.as_deref().unwrap_or_default(),
),
}
}
Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id), Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id),
}; };
truncate_chars(&text, max_bytes) truncate_chars(&text, max_bytes)
@@ -633,6 +652,38 @@ mod tests {
); );
} }
#[test]
fn tool_image_projection_exposes_only_bounded_metadata() {
let view = SessionCapture::new(
"segment-1",
vec![Item::tool_result_item_with_attachments(
"c1",
"attached",
None,
false,
vec![llm_engine::tool::Attachment::Image(
llm_engine::tool::ImageAttachment::new(
"image/png",
b"private-image-body".to_vec(),
),
)],
)],
);
let result = view.read(
ReadSelector::Id("E00000000"),
ReadOptions {
include_tools: true,
detail: ReadDetail::Full,
..ReadOptions::default()
},
);
assert_eq!(result.entries.len(), 1);
assert!(result.entries[0].text.contains("attachments: 1 image(s)"));
assert!(!result.entries[0].text.contains("private-image-body"));
assert!(!result.entries[0].text.contains("cHJpdmF0ZS"));
}
#[test] #[test]
fn system_prompt_and_reasoning_are_excluded_from_every_projection() { fn system_prompt_and_reasoning_are_excluded_from_every_projection() {
let view = SessionCapture::new( let view = SessionCapture::new(
+2
View File
@@ -6507,6 +6507,7 @@ mod build_summary_prompt_tests {
call_id: "call-1".into(), call_id: "call-1".into(),
summary: "wrote a file".into(), summary: "wrote a file".into(),
content: None, content: None,
attachments: Vec::new(),
is_error: false, is_error: false,
}, },
}, },
@@ -6548,6 +6549,7 @@ mod build_summary_prompt_tests {
call_id: "call-1".into(), call_id: "call-1".into(),
summary: "wrote a file".into(), summary: "wrote a file".into(),
content: None, content: None,
attachments: Vec::new(),
is_error: false, is_error: false,
}, },
}, },
@@ -0,0 +1,49 @@
# Request-local image injection bypassed durable history and pruning
Date: 2026-08-06
## Symptom
The first `ViewImage` implementation stored image bytes in `ToolOutput.attachments` with
`#[serde(skip)]`, projected them into a synthetic user image only while building one provider
request, and then cleared the attachment from Worker history. The model could therefore answer
from pixels that disappeared from the next turn, session restore, pruning, and compaction
observation.
## Why this was wrong
Yoi's context policy requires new model-visible input to be appended and committed to
`worker.history` before request construction. Pure pruning may alter a request-context clone
because the projection is deterministic from durable history; request-local input injection is
not equivalent.
The transient image also changed the middle of the next prompt prefix without an explicit prune,
which reduced prompt-cache reuse and left later turns without the evidence behind the assistant
response.
Local reference review confirmed the intended pattern:
- Codex records `view_image` as image-bearing `FunctionCallOutput` content and explicitly tests
that no separate image message is injected.
- OpenCode persists image file parts in Session state and makes media removal an explicit
compaction/pruning decision.
## Resolution
Image attachments are now durable ToolResult detail:
- Image bytes are base64-serialized through normal Item/session-log persistence.
- OpenAI Responses lowers them to `function_call_output` content items.
- OpenAI Chat deterministically lowers the durable ToolResult to tool text followed by a user
image message while preserving parallel-tool-result ordering.
- The existing ToolResult pruning projection removes both text detail and image attachments from
the request-context clone while retaining the summary and original persistent history.
- Session exploration and compaction-facing projections expose only bounded attachment metadata,
never raw base64 as ordinary text.
## Guardrail
Do not use `#[serde(skip)]`, post-build clearing, or one-shot synthetic messages for input that can
influence model output. Provider-specific synthetic messages are acceptable only when they are a
stable projection of a committed history item and are regenerated identically until an explicit
pruning or compaction boundary.