fix: make image tool results durably prunable
This commit is contained in:
Generated
+1
@@ -3895,6 +3895,7 @@ name = "session-store"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
"base64 0.22.1",
|
||||||
"futures",
|
"futures",
|
||||||
"llm-engine",
|
"llm-engine",
|
||||||
"protocol",
|
"protocol",
|
||||||
|
|||||||
@@ -19,19 +19,15 @@ use crate::{
|
|||||||
},
|
},
|
||||||
llm_client::{
|
llm_client::{
|
||||||
ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream,
|
ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream,
|
||||||
ToolDefinition,
|
ToolDefinition, error::is_retryable, event::Event, retry::RetryPolicy,
|
||||||
error::is_retryable,
|
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT, types::parse_tool_arguments,
|
||||||
event::Event,
|
|
||||||
retry::RetryPolicy,
|
|
||||||
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT,
|
|
||||||
types::{ContentPart, parse_tool_arguments},
|
|
||||||
},
|
},
|
||||||
state::{EngineState, Locked, Mutable},
|
state::{EngineState, Locked, Mutable},
|
||||||
timeline::event::{ErrorEvent, StatusEvent, UsageEvent},
|
timeline::event::{ErrorEvent, StatusEvent, UsageEvent},
|
||||||
timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector},
|
timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector},
|
||||||
tool::{
|
tool::{
|
||||||
Attachment, ToolCall, ToolDefinition as EngineToolDefinition, ToolError,
|
ToolCall, ToolDefinition as EngineToolDefinition, ToolError, ToolExecutionContext,
|
||||||
ToolExecutionContext, ToolOutputLimits, ToolResult, truncate_content,
|
ToolOutputLimits, ToolResult, truncate_content,
|
||||||
},
|
},
|
||||||
tool_server::{ToolServer, ToolServerHandle},
|
tool_server::{ToolServer, ToolServerHandle},
|
||||||
};
|
};
|
||||||
@@ -159,33 +155,6 @@ enum StreamCompletion {
|
|||||||
Interrupted { reason: String },
|
Interrupted { reason: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
fn project_transient_attachments(items: &mut Vec<Item>) {
|
|
||||||
let mut projected = Vec::with_capacity(items.len());
|
|
||||||
let mut pending_parts = Vec::new();
|
|
||||||
|
|
||||||
for mut item in items.drain(..) {
|
|
||||||
let is_tool_result = matches!(item, Item::ToolResult { .. });
|
|
||||||
if !is_tool_result && !pending_parts.is_empty() {
|
|
||||||
projected.push(Item::user_message_parts(std::mem::take(&mut pending_parts)));
|
|
||||||
}
|
|
||||||
if let Item::ToolResult { attachments, .. } = &item {
|
|
||||||
for attachment in attachments {
|
|
||||||
let Attachment::Image(image) = attachment;
|
|
||||||
pending_parts.push(ContentPart::image(
|
|
||||||
image.mime_type(),
|
|
||||||
Arc::from(image.data()),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
item.clear_transient_attachments();
|
|
||||||
projected.push(item);
|
|
||||||
}
|
|
||||||
if !pending_parts.is_empty() {
|
|
||||||
projected.push(Item::user_message_parts(pending_parts));
|
|
||||||
}
|
|
||||||
*items = projected;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
|
pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
|
||||||
/// LLM client
|
/// LLM client
|
||||||
client: C,
|
client: C,
|
||||||
@@ -1280,8 +1249,6 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
cb(current_llm_call);
|
cb(current_llm_call);
|
||||||
}
|
}
|
||||||
|
|
||||||
project_transient_attachments(&mut request_context);
|
|
||||||
|
|
||||||
// Stream LLM response
|
// Stream LLM response
|
||||||
self.emit_lifecycle_trace(
|
self.emit_lifecycle_trace(
|
||||||
current_turn,
|
current_turn,
|
||||||
@@ -1290,12 +1257,6 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
items_trace_payload(&request_context, tool_definitions.len(), None, false),
|
items_trace_payload(&request_context, tool_definitions.len(), None, false),
|
||||||
);
|
);
|
||||||
let request = self.build_request(&tool_definitions, &request_context);
|
let request = self.build_request(&tool_definitions, &request_context);
|
||||||
// Structured attachments are single-use provider payloads. `request`
|
|
||||||
// owns its Arc-backed copy; history is cleared before any later turn
|
|
||||||
// can persist or resend the binary body.
|
|
||||||
for item in &mut self.history {
|
|
||||||
item.clear_transient_attachments();
|
|
||||||
}
|
|
||||||
self.emit_lifecycle_trace(
|
self.emit_lifecycle_trace(
|
||||||
current_turn,
|
current_turn,
|
||||||
current_llm_call,
|
current_llm_call,
|
||||||
@@ -2199,48 +2160,36 @@ fn item_kind(item: &Item) -> &'static str {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::llm_client::types::Role;
|
use crate::tool::{Attachment, ImageAttachment};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn transient_tool_attachment_becomes_non_persistent_user_image_part() {
|
fn tool_attachment_round_trips_through_durable_history_json() {
|
||||||
let body: Arc<[u8]> = Arc::from(&b"secret-image-body"[..]);
|
let body: Arc<[u8]> = Arc::from(&b"image-body"[..]);
|
||||||
let mut items = vec![Item::tool_result_item_with_attachments(
|
let items = vec![Item::tool_result_item_with_attachments(
|
||||||
"call_image",
|
"call_image",
|
||||||
"attached",
|
"attached",
|
||||||
None,
|
None,
|
||||||
false,
|
false,
|
||||||
vec![Attachment::Image(crate::tool::ImageAttachment::new(
|
vec![Attachment::Image(ImageAttachment::new(
|
||||||
"image/png",
|
"image/png",
|
||||||
body.clone(),
|
body.clone(),
|
||||||
))],
|
))],
|
||||||
)];
|
)];
|
||||||
|
|
||||||
project_transient_attachments(&mut items);
|
|
||||||
|
|
||||||
assert!(matches!(
|
|
||||||
&items[0],
|
|
||||||
Item::ToolResult { attachments, .. } if attachments.is_empty()
|
|
||||||
));
|
|
||||||
match &items[1] {
|
|
||||||
Item::Message {
|
|
||||||
role: Role::User,
|
|
||||||
content,
|
|
||||||
..
|
|
||||||
} => match &content[0] {
|
|
||||||
ContentPart::Image { media_type, source } => {
|
|
||||||
assert_eq!(media_type, "image/png");
|
|
||||||
assert_eq!(source.data(), body.as_ref());
|
|
||||||
assert_eq!(source.bytes(), body.len());
|
|
||||||
assert_eq!(content[0].as_text(), "[image attachment omitted]");
|
|
||||||
}
|
|
||||||
other => panic!("unexpected content part: {other:?}"),
|
|
||||||
},
|
|
||||||
other => panic!("unexpected projected item: {other:?}"),
|
|
||||||
}
|
|
||||||
let persisted = serde_json::to_string(&items).unwrap();
|
let persisted = serde_json::to_string(&items).unwrap();
|
||||||
assert!(!persisted.contains("secret-image-body"));
|
assert!(persisted.contains("aW1hZ2UtYm9keQ=="));
|
||||||
assert!(!persisted.contains("base64"));
|
let restored: Vec<Item> = serde_json::from_str(&persisted).unwrap();
|
||||||
|
assert_eq!(restored, items);
|
||||||
|
assert!(matches!(
|
||||||
|
&restored[0],
|
||||||
|
Item::ToolResult { attachments, .. }
|
||||||
|
if matches!(
|
||||||
|
attachments.as_slice(),
|
||||||
|
[Attachment::Image(image)]
|
||||||
|
if image.mime_type() == "image/png" && image.data() == body.as_ref()
|
||||||
|
)
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -268,9 +268,6 @@ impl AnthropicScheme {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|p| match p {
|
.map(|p| match p {
|
||||||
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
|
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
|
||||||
ContentPart::Image { .. } => {
|
|
||||||
AnthropicContentPart::text(p.as_text().to_string())
|
|
||||||
}
|
|
||||||
ContentPart::Refusal { refusal } => {
|
ContentPart::Refusal { refusal } => {
|
||||||
AnthropicContentPart::text(refusal.clone())
|
AnthropicContentPart::text(refusal.clone())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,10 +5,13 @@
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::llm_client::{
|
use crate::{
|
||||||
|
llm_client::{
|
||||||
Request,
|
Request,
|
||||||
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
||||||
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
|
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
|
||||||
|
},
|
||||||
|
tool::Attachment,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::OpenAIScheme;
|
use super::OpenAIScheme;
|
||||||
@@ -185,6 +188,21 @@ impl OpenAIScheme {
|
|||||||
/// - Assistant messages have role "assistant"
|
/// - Assistant messages have role "assistant"
|
||||||
/// - Tool calls are within assistant messages as tool_calls array
|
/// - Tool calls are within assistant messages as tool_calls array
|
||||||
/// - Tool results have role "tool" with tool_call_id
|
/// - Tool results have role "tool" with tool_call_id
|
||||||
|
fn flush_pending_tool_result_images(
|
||||||
|
messages: &mut Vec<OpenAIMessage>,
|
||||||
|
pending_images: &mut Vec<OpenAIContentPart>,
|
||||||
|
) {
|
||||||
|
if !pending_images.is_empty() {
|
||||||
|
messages.push(OpenAIMessage {
|
||||||
|
role: "user".to_string(),
|
||||||
|
content: Some(OpenAIContent::Parts(std::mem::take(pending_images))),
|
||||||
|
tool_calls: vec![],
|
||||||
|
tool_call_id: None,
|
||||||
|
name: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn convert_items_to_messages(
|
fn convert_items_to_messages(
|
||||||
&self,
|
&self,
|
||||||
items: &[Item],
|
items: &[Item],
|
||||||
@@ -193,8 +211,15 @@ impl OpenAIScheme {
|
|||||||
let mut messages = Vec::new();
|
let mut messages = Vec::new();
|
||||||
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
|
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
|
||||||
let mut pending_assistant_text: Option<String> = None;
|
let mut pending_assistant_text: Option<String> = None;
|
||||||
|
let mut pending_tool_result_images: Vec<OpenAIContentPart> = Vec::new();
|
||||||
|
|
||||||
for item in items {
|
for item in items {
|
||||||
|
if !matches!(item, Item::ToolResult { .. }) {
|
||||||
|
Self::flush_pending_tool_result_images(
|
||||||
|
&mut messages,
|
||||||
|
&mut pending_tool_result_images,
|
||||||
|
);
|
||||||
|
}
|
||||||
match item {
|
match item {
|
||||||
Item::Message { role, content, .. } => {
|
Item::Message { role, content, .. } => {
|
||||||
// Flush pending tool calls
|
// Flush pending tool calls
|
||||||
@@ -209,41 +234,13 @@ impl OpenAIScheme {
|
|||||||
Role::Assistant => "assistant",
|
Role::Assistant => "assistant",
|
||||||
Role::System => "system",
|
Role::System => "system",
|
||||||
};
|
};
|
||||||
let has_image = matches!(role, Role::User)
|
let message_content = OpenAIContent::Text(
|
||||||
&& supports_images
|
|
||||||
&& content
|
|
||||||
.iter()
|
|
||||||
.any(|part| matches!(part, ContentPart::Image { .. }));
|
|
||||||
let message_content = if has_image {
|
|
||||||
OpenAIContent::Parts(
|
|
||||||
content
|
|
||||||
.iter()
|
|
||||||
.map(|part| match part {
|
|
||||||
ContentPart::Text { text } => {
|
|
||||||
OpenAIContentPart::Text { text: text.clone() }
|
|
||||||
}
|
|
||||||
ContentPart::Image { media_type, source } => {
|
|
||||||
OpenAIContentPart::ImageUrl {
|
|
||||||
image_url: ImageUrl {
|
|
||||||
url: image_data_url(media_type, source.data()),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ContentPart::Refusal { refusal } => OpenAIContentPart::Text {
|
|
||||||
text: refusal.clone(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
OpenAIContent::Text(
|
|
||||||
content
|
content
|
||||||
.iter()
|
.iter()
|
||||||
.map(ContentPart::as_text)
|
.map(ContentPart::as_text)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(""),
|
.join(""),
|
||||||
)
|
);
|
||||||
};
|
|
||||||
|
|
||||||
messages.push(OpenAIMessage {
|
messages.push(OpenAIMessage {
|
||||||
role: openai_role.to_string(),
|
role: openai_role.to_string(),
|
||||||
@@ -277,19 +274,35 @@ impl OpenAIScheme {
|
|||||||
call_id,
|
call_id,
|
||||||
summary,
|
summary,
|
||||||
content,
|
content,
|
||||||
|
attachments,
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
// Flush pending tool calls before tool result
|
// OpenAI requires every parallel tool result before a new user message.
|
||||||
self.flush_pending_assistant(
|
self.flush_pending_assistant(
|
||||||
&mut messages,
|
&mut messages,
|
||||||
&mut pending_tool_calls,
|
&mut pending_tool_calls,
|
||||||
&mut pending_assistant_text,
|
&mut pending_assistant_text,
|
||||||
);
|
);
|
||||||
|
|
||||||
let text = match content {
|
let mut text = match content {
|
||||||
Some(c) => format!("{summary}\n{c}"),
|
Some(c) => format!("{summary}\n{c}"),
|
||||||
None => summary.clone(),
|
None => summary.clone(),
|
||||||
};
|
};
|
||||||
|
if supports_images {
|
||||||
|
pending_tool_result_images.extend(attachments.iter().map(|attachment| {
|
||||||
|
let Attachment::Image(image) = attachment;
|
||||||
|
OpenAIContentPart::ImageUrl {
|
||||||
|
image_url: ImageUrl {
|
||||||
|
url: image_data_url(image.mime_type(), image.data()),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
} else if !attachments.is_empty() {
|
||||||
|
text.push_str(&format!(
|
||||||
|
"\n[{} image attachment(s) omitted: model does not support images]",
|
||||||
|
attachments.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
messages.push(OpenAIMessage {
|
messages.push(OpenAIMessage {
|
||||||
role: "tool".to_string(),
|
role: "tool".to_string(),
|
||||||
content: Some(OpenAIContent::Text(text)),
|
content: Some(OpenAIContent::Text(text)),
|
||||||
@@ -317,6 +330,7 @@ impl OpenAIScheme {
|
|||||||
&mut pending_tool_calls,
|
&mut pending_tool_calls,
|
||||||
&mut pending_assistant_text,
|
&mut pending_assistant_text,
|
||||||
);
|
);
|
||||||
|
Self::flush_pending_tool_result_images(&mut messages, &mut pending_tool_result_images);
|
||||||
|
|
||||||
messages
|
messages
|
||||||
}
|
}
|
||||||
@@ -481,28 +495,27 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parallel_tool_results_precede_synthetic_image_message() {
|
fn parallel_tool_results_precede_durable_image_projection() {
|
||||||
let scheme = OpenAIScheme::new();
|
let scheme = OpenAIScheme::new();
|
||||||
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
||||||
let request = Request::new()
|
let request = Request::new()
|
||||||
.item(Item::tool_call("call_image", "ViewImage", "{}"))
|
.item(Item::tool_call("call_image", "ViewImage", "{}"))
|
||||||
.item(Item::tool_call("call_text", "Read", "{}"))
|
.item(Item::tool_call("call_text", "Read", "{}"))
|
||||||
.item(Item::tool_result_item(
|
.item(Item::tool_result_item_with_attachments(
|
||||||
"call_image",
|
"call_image",
|
||||||
"Attached image",
|
"Attached image",
|
||||||
None,
|
None,
|
||||||
false,
|
false,
|
||||||
|
vec![crate::tool::Attachment::Image(
|
||||||
|
crate::tool::ImageAttachment::new("image/png", image),
|
||||||
|
)],
|
||||||
))
|
))
|
||||||
.item(Item::tool_result_item(
|
.item(Item::tool_result_item(
|
||||||
"call_text",
|
"call_text",
|
||||||
"Read text",
|
"Read text",
|
||||||
None,
|
None,
|
||||||
false,
|
false,
|
||||||
))
|
));
|
||||||
.item(Item::user_message_parts(vec![ContentPart::image(
|
|
||||||
"image/png",
|
|
||||||
image,
|
|
||||||
)]));
|
|
||||||
let json = serde_json::to_value(
|
let json = serde_json::to_value(
|
||||||
&scheme
|
&scheme
|
||||||
.build_request("gpt-4o", &request, &vision_cap())
|
.build_request("gpt-4o", &request, &vision_cap())
|
||||||
@@ -518,7 +531,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn tool_image_is_structured_as_following_user_content_without_persisting_bytes() {
|
fn durable_tool_image_is_deterministically_lowered_to_following_user_content() {
|
||||||
let scheme = OpenAIScheme::new();
|
let scheme = OpenAIScheme::new();
|
||||||
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
||||||
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
|
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
|
||||||
@@ -533,8 +546,8 @@ mod tests {
|
|||||||
vec![attachment],
|
vec![attachment],
|
||||||
);
|
);
|
||||||
let persisted = serde_json::to_string(&item).unwrap();
|
let persisted = serde_json::to_string(&item).unwrap();
|
||||||
assert!(!persisted.contains("base64"));
|
assert!(persisted.contains("attachments"));
|
||||||
assert!(!persisted.contains("attachments"));
|
let restored: Item = serde_json::from_str(&persisted).unwrap();
|
||||||
|
|
||||||
let request = Request::new()
|
let request = Request::new()
|
||||||
.item(Item::tool_call(
|
.item(Item::tool_call(
|
||||||
@@ -542,13 +555,16 @@ mod tests {
|
|||||||
"ViewImage",
|
"ViewImage",
|
||||||
r#"{"path":"a.png"}"#,
|
r#"{"path":"a.png"}"#,
|
||||||
))
|
))
|
||||||
.item(item)
|
.item(restored);
|
||||||
.item(Item::user_message_parts(vec![ContentPart::image(
|
|
||||||
"image/png",
|
|
||||||
image,
|
|
||||||
)]));
|
|
||||||
let body = scheme.build_request("gpt-4o", &request, &vision_cap());
|
let body = scheme.build_request("gpt-4o", &request, &vision_cap());
|
||||||
let json = serde_json::to_value(&body.messages).unwrap();
|
let json = serde_json::to_value(&body.messages).unwrap();
|
||||||
|
let rebuilt = serde_json::to_value(
|
||||||
|
&scheme
|
||||||
|
.build_request("gpt-4o", &request, &vision_cap())
|
||||||
|
.messages,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(rebuilt, json);
|
||||||
|
|
||||||
assert_eq!(json[0]["role"], "assistant");
|
assert_eq!(json[0]["role"], "assistant");
|
||||||
assert_eq!(json[1]["role"], "tool");
|
assert_eq!(json[1]["role"], "tool");
|
||||||
|
|||||||
@@ -7,14 +7,31 @@
|
|||||||
use serde::{Serialize, Serializer};
|
use serde::{Serialize, Serializer};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::llm_client::{
|
use crate::{
|
||||||
|
llm_client::{
|
||||||
Request,
|
Request,
|
||||||
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
||||||
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
|
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
|
||||||
|
},
|
||||||
|
tool::Attachment,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::OpenAIResponsesScheme;
|
use super::OpenAIResponsesScheme;
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub(crate) enum FunctionCallOutputBody {
|
||||||
|
Text(String),
|
||||||
|
ContentItems(Vec<FunctionCallOutputContentItem>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
#[serde(tag = "type", rename_all = "snake_case")]
|
||||||
|
pub(crate) enum FunctionCallOutputContentItem {
|
||||||
|
InputText { text: String },
|
||||||
|
InputImage { image_url: String },
|
||||||
|
}
|
||||||
|
|
||||||
/// `/v1/responses` のリクエスト body。
|
/// `/v1/responses` のリクエスト body。
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
pub(crate) struct ResponsesRequest {
|
pub(crate) struct ResponsesRequest {
|
||||||
@@ -89,7 +106,10 @@ pub(crate) enum InputItem {
|
|||||||
arguments: String,
|
arguments: String,
|
||||||
},
|
},
|
||||||
/// function tool の結果(user 側)。
|
/// function tool の結果(user 側)。
|
||||||
FunctionCallOutput { call_id: String, output: String },
|
FunctionCallOutput {
|
||||||
|
call_id: String,
|
||||||
|
output: FunctionCallOutputBody,
|
||||||
|
},
|
||||||
/// reasoning item。`encrypted_content` があれば必ず添える。
|
/// reasoning item。`encrypted_content` があれば必ず添える。
|
||||||
Reasoning {
|
Reasoning {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
@@ -114,10 +134,6 @@ pub(crate) enum InputContent {
|
|||||||
/// user / developer 側のテキスト
|
/// user / developer 側のテキスト
|
||||||
InputText { text: String },
|
InputText { text: String },
|
||||||
/// user 側の画像
|
/// user 側の画像
|
||||||
InputImage {
|
|
||||||
image_url: String,
|
|
||||||
detail: &'static str,
|
|
||||||
},
|
|
||||||
/// assistant 側のテキスト
|
/// assistant 側のテキスト
|
||||||
OutputText { text: String },
|
OutputText { text: String },
|
||||||
}
|
}
|
||||||
@@ -249,15 +265,6 @@ fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputIte
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|part| match part {
|
.map(|part| match part {
|
||||||
ContentPart::Text { text } => text_variant(text.clone()),
|
ContentPart::Text { text } => text_variant(text.clone()),
|
||||||
ContentPart::Image { media_type, source }
|
|
||||||
if matches!(role, Role::User) && supports_images =>
|
|
||||||
{
|
|
||||||
InputContent::InputImage {
|
|
||||||
image_url: image_data_url(media_type, source.data()),
|
|
||||||
detail: "auto",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ContentPart::Image { .. } => text_variant(part.as_text().to_string()),
|
|
||||||
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
|
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -284,12 +291,30 @@ fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputIte
|
|||||||
call_id,
|
call_id,
|
||||||
summary,
|
summary,
|
||||||
content,
|
content,
|
||||||
|
attachments,
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
let output = match content {
|
let text = match content {
|
||||||
Some(c) => format!("{summary}\n{c}"),
|
Some(c) => format!("{summary}\n{c}"),
|
||||||
None => summary.clone(),
|
None => summary.clone(),
|
||||||
};
|
};
|
||||||
|
let output = if attachments.is_empty() {
|
||||||
|
FunctionCallOutputBody::Text(text)
|
||||||
|
} else if supports_images {
|
||||||
|
let mut parts = vec![FunctionCallOutputContentItem::InputText { text }];
|
||||||
|
parts.extend(attachments.iter().map(|attachment| {
|
||||||
|
let Attachment::Image(image) = attachment;
|
||||||
|
FunctionCallOutputContentItem::InputImage {
|
||||||
|
image_url: image_data_url(image.mime_type(), image.data()),
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
FunctionCallOutputBody::ContentItems(parts)
|
||||||
|
} else {
|
||||||
|
FunctionCallOutputBody::Text(format!(
|
||||||
|
"{text}\n[{} image attachment(s) omitted: model does not support images]",
|
||||||
|
attachments.len()
|
||||||
|
))
|
||||||
|
};
|
||||||
out.push(InputItem::FunctionCallOutput {
|
out.push(InputItem::FunctionCallOutput {
|
||||||
call_id: call_id.clone(),
|
call_id: call_id.clone(),
|
||||||
output,
|
output,
|
||||||
@@ -701,7 +726,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn synthetic_user_image_uses_responses_input_image_content() {
|
fn durable_tool_image_uses_function_call_output_content_items() {
|
||||||
let scheme = OpenAIResponsesScheme::new();
|
let scheme = OpenAIResponsesScheme::new();
|
||||||
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
||||||
let item = Item::tool_result_item_with_attachments(
|
let item = Item::tool_result_item_with_attachments(
|
||||||
@@ -710,32 +735,35 @@ mod tests {
|
|||||||
None,
|
None,
|
||||||
false,
|
false,
|
||||||
vec![crate::tool::Attachment::Image(
|
vec![crate::tool::Attachment::Image(
|
||||||
crate::tool::ImageAttachment::new("image/png", image.clone()),
|
crate::tool::ImageAttachment::new("image/png", image),
|
||||||
)],
|
)],
|
||||||
);
|
);
|
||||||
|
let persisted = serde_json::to_string(&item).unwrap();
|
||||||
|
let restored: Item = serde_json::from_str(&persisted).unwrap();
|
||||||
let req = Request::new()
|
let req = Request::new()
|
||||||
.item(Item::tool_call(
|
.item(Item::tool_call(
|
||||||
"call_image",
|
"call_image",
|
||||||
"ViewImage",
|
"ViewImage",
|
||||||
r#"{"path":"a.png"}"#,
|
r#"{"path":"a.png"}"#,
|
||||||
))
|
))
|
||||||
.item(item)
|
.item(restored);
|
||||||
.item(Item::user_message_parts(vec![ContentPart::image(
|
|
||||||
"image/png",
|
|
||||||
image,
|
|
||||||
)]));
|
|
||||||
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
|
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
|
||||||
let json = serde_json::to_value(&body).unwrap();
|
let json = serde_json::to_value(&body).unwrap();
|
||||||
|
|
||||||
assert_eq!(json["input"][1]["type"], "function_call_output");
|
assert_eq!(json["input"][1]["type"], "function_call_output");
|
||||||
assert_eq!(json["input"][2]["type"], "message");
|
assert_eq!(json["input"].as_array().unwrap().len(), 2);
|
||||||
assert_eq!(json["input"][2]["content"][0]["type"], "input_image");
|
assert_eq!(json["input"][1]["output"][0]["type"], "input_text");
|
||||||
|
assert_eq!(json["input"][1]["output"][1]["type"], "input_image");
|
||||||
assert!(
|
assert!(
|
||||||
json["input"][2]["content"][0]["image_url"]
|
json["input"][1]["output"][1]["image_url"]
|
||||||
.as_str()
|
.as_str()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.starts_with("data:image/png;base64,")
|
.starts_with("data:image/png;base64,")
|
||||||
);
|
);
|
||||||
|
let rebuilt =
|
||||||
|
serde_json::to_value(scheme.build_request("gpt-5", &req, &cap_with_reasoning()))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(rebuilt["input"], json["input"]);
|
||||||
|
|
||||||
let mut no_vision = cap_with_reasoning();
|
let mut no_vision = cap_with_reasoning();
|
||||||
no_vision.vision = false;
|
no_vision.vision = false;
|
||||||
|
|||||||
@@ -124,8 +124,8 @@ pub enum Item {
|
|||||||
/// Whether the tool result represents an execution error.
|
/// Whether the tool result represents an execution error.
|
||||||
#[serde(default, skip_serializing_if = "is_false")]
|
#[serde(default, skip_serializing_if = "is_false")]
|
||||||
is_error: bool,
|
is_error: bool,
|
||||||
/// Request-local structured payloads. Never serialized or persisted.
|
/// Durable binary details (removed with `content` by normal pruning).
|
||||||
#[serde(skip, default)]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
attachments: Vec<Attachment>,
|
attachments: Vec<Attachment>,
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -264,7 +264,7 @@ impl Item {
|
|||||||
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
|
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a tool result item with request-local structured attachments.
|
/// Create a tool result item with durable, prunable structured attachments.
|
||||||
pub fn tool_result_item_with_attachments(
|
pub fn tool_result_item_with_attachments(
|
||||||
call_id: impl Into<String>,
|
call_id: impl Into<String>,
|
||||||
summary: impl Into<String>,
|
summary: impl Into<String>,
|
||||||
@@ -282,13 +282,6 @@ impl Item {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drop request-local attachments after constructing the provider request.
|
|
||||||
pub fn clear_transient_attachments(&mut self) {
|
|
||||||
if let Self::ToolResult { attachments, .. } = self {
|
|
||||||
attachments.clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create a tool result item with summary and content.
|
/// Create a tool result item with summary and content.
|
||||||
pub fn tool_result_with_content(
|
pub fn tool_result_with_content(
|
||||||
call_id: impl Into<String>,
|
call_id: impl Into<String>,
|
||||||
@@ -457,38 +450,6 @@ pub fn parse_tool_arguments(arguments: &str) -> serde_json::Value {
|
|||||||
// Content Parts - Components within message items
|
// Content Parts - Components within message items
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
|
|
||||||
pub struct ImageSource {
|
|
||||||
bytes: usize,
|
|
||||||
#[serde(skip, default)]
|
|
||||||
data: Arc<[u8]>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ImageSource {
|
|
||||||
pub fn new(data: Arc<[u8]>) -> Self {
|
|
||||||
Self {
|
|
||||||
bytes: data.len(),
|
|
||||||
data,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn data(&self) -> &[u8] {
|
|
||||||
&self.data
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn bytes(&self) -> usize {
|
|
||||||
self.bytes
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl fmt::Debug for ImageSource {
|
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
||||||
f.debug_struct("ImageSource")
|
|
||||||
.field("bytes", &self.bytes)
|
|
||||||
.finish()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Content part within a message item
|
/// Content part within a message item
|
||||||
///
|
///
|
||||||
/// Text content is role-agnostic; the containing Item's Role determines
|
/// Text content is role-agnostic; the containing Item's Role determines
|
||||||
@@ -502,12 +463,6 @@ pub enum ContentPart {
|
|||||||
text: String,
|
text: String,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Request-local image content. The source bytes are never serialized.
|
|
||||||
Image {
|
|
||||||
media_type: String,
|
|
||||||
source: ImageSource,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Refusal content (for assistant messages)
|
/// Refusal content (for assistant messages)
|
||||||
Refusal {
|
Refusal {
|
||||||
/// The refusal message
|
/// The refusal message
|
||||||
@@ -528,20 +483,10 @@ impl ContentPart {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn image(media_type: impl Into<String>, data: Arc<[u8]>) -> Self {
|
/// Get a textual projection of the content part.
|
||||||
Self::Image {
|
|
||||||
media_type: media_type.into(),
|
|
||||||
source: ImageSource::new(data),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get a bounded textual projection. Image content is represented by an
|
|
||||||
/// explicit placeholder rather than silently becoming an empty string,
|
|
||||||
/// filesystem path, data URL, or base64 body.
|
|
||||||
pub fn as_text(&self) -> &str {
|
pub fn as_text(&self) -> &str {
|
||||||
match self {
|
match self {
|
||||||
Self::Text { text } => text,
|
Self::Text { text } => text,
|
||||||
Self::Image { .. } => "[image attachment omitted]",
|
|
||||||
Self::Refusal { refusal } => refusal,
|
Self::Refusal { refusal } => refusal,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -110,25 +110,30 @@ impl Default for PruneConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set `content = None` on each `Item::ToolResult` at the given indices.
|
/// Remove detailed text and attachments from each `Item::ToolResult` at the given indices.
|
||||||
///
|
///
|
||||||
/// Returns the number of items that were actually modified — items that
|
/// The mandatory summary remains. Returns the number of items that were actually
|
||||||
/// are already content-less are counted as 0. Intended for use on a
|
/// modified — results that already contain no detail are counted as 0. Intended
|
||||||
/// request-context clone (never on a persistent history).
|
/// for use on a request-context clone (never on a persistent history).
|
||||||
pub fn project(items: &mut [Item], indices: &[usize]) -> usize {
|
pub fn project(items: &mut [Item], indices: &[usize]) -> usize {
|
||||||
let mut count = 0;
|
let mut count = 0;
|
||||||
for &i in indices {
|
for &i in indices {
|
||||||
if let Item::ToolResult { content, .. } = &mut items[i]
|
if let Item::ToolResult {
|
||||||
&& content.is_some()
|
content,
|
||||||
|
attachments,
|
||||||
|
..
|
||||||
|
} = &mut items[i]
|
||||||
|
&& (content.is_some() || !attachments.is_empty())
|
||||||
{
|
{
|
||||||
*content = None;
|
*content = None;
|
||||||
|
attachments.clear();
|
||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
count
|
count
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Indices of `Item::ToolResult { content: Some(_), .. }` that lie before
|
/// Indices of detailed `Item::ToolResult` values that lie before
|
||||||
/// the suffix protected by `protected_tokens`. Pure: does not mutate `items`.
|
/// the suffix protected by `protected_tokens`. Pure: does not mutate `items`.
|
||||||
///
|
///
|
||||||
/// Returns an empty vector when token estimates are unavailable (`NoData`) or
|
/// Returns an empty vector when token estimates are unavailable (`NoData`) or
|
||||||
@@ -159,8 +164,10 @@ pub fn evaluate_candidates(
|
|||||||
.enumerate()
|
.enumerate()
|
||||||
.filter_map(|(i, item)| match item {
|
.filter_map(|(i, item)| match item {
|
||||||
Item::ToolResult {
|
Item::ToolResult {
|
||||||
content: Some(_), ..
|
content,
|
||||||
} => Some(i),
|
attachments,
|
||||||
|
..
|
||||||
|
} if content.is_some() || !attachments.is_empty() => Some(i),
|
||||||
_ => None,
|
_ => None,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -373,6 +380,38 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn project_drops_image_detail_but_keeps_summary_and_persistent_source() {
|
||||||
|
let original = vec![Item::tool_result_item_with_attachments(
|
||||||
|
"call_image",
|
||||||
|
"Attached image/png image (12 bytes)",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
vec![crate::tool::Attachment::Image(
|
||||||
|
crate::tool::ImageAttachment::new("image/png", b"image-body".to_vec()),
|
||||||
|
)],
|
||||||
|
)];
|
||||||
|
let mut request_context = original.clone();
|
||||||
|
|
||||||
|
let estimates = uniform_estimates(&original, 100);
|
||||||
|
assert_eq!(prunable_indices(&original, 0, &estimates), vec![0]);
|
||||||
|
|
||||||
|
assert_eq!(project(&mut request_context, &[0]), 1);
|
||||||
|
assert!(matches!(
|
||||||
|
&request_context[0],
|
||||||
|
Item::ToolResult {
|
||||||
|
summary,
|
||||||
|
content: None,
|
||||||
|
attachments,
|
||||||
|
..
|
||||||
|
} if summary == "Attached image/png image (12 bytes)" && attachments.is_empty()
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
&original[0],
|
||||||
|
Item::ToolResult { attachments, .. } if attachments.len() == 1
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn project_skips_already_pruned_items() {
|
fn project_skips_already_pruned_items() {
|
||||||
// indices points at an item whose content is already None.
|
// indices points at an item whose content is already None.
|
||||||
|
|||||||
@@ -6,7 +6,8 @@
|
|||||||
use std::{collections::HashMap, fmt, sync::Arc};
|
use std::{collections::HashMap, fmt, sync::Arc};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||||
|
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
@@ -120,12 +121,39 @@ impl fmt::Debug for ImageAttachment {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Request-local binary payload emitted by a tool.
|
#[derive(Serialize, Deserialize)]
|
||||||
///
|
struct ImageAttachmentWire {
|
||||||
/// Attachments are deliberately excluded from serde. They may be projected into
|
mime_type: String,
|
||||||
/// the immediately following provider request, but never into persisted history,
|
data: String,
|
||||||
/// protocol events, logs, or telemetry.
|
}
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
|
impl Serialize for ImageAttachment {
|
||||||
|
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||||
|
where
|
||||||
|
S: Serializer,
|
||||||
|
{
|
||||||
|
ImageAttachmentWire {
|
||||||
|
mime_type: self.mime_type.clone(),
|
||||||
|
data: STANDARD.encode(self.data.as_ref()),
|
||||||
|
}
|
||||||
|
.serialize(serializer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de> Deserialize<'de> for ImageAttachment {
|
||||||
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let wire = ImageAttachmentWire::deserialize(deserializer)?;
|
||||||
|
let data = STANDARD.decode(wire.data).map_err(D::Error::custom)?;
|
||||||
|
Ok(Self::new(wire.mime_type, data))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Durable binary detail emitted by a tool and handled by normal ToolResult pruning.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(tag = "type", content = "payload", rename_all = "snake_case")]
|
||||||
pub enum Attachment {
|
pub enum Attachment {
|
||||||
Image(ImageAttachment),
|
Image(ImageAttachment),
|
||||||
}
|
}
|
||||||
@@ -133,8 +161,8 @@ pub enum Attachment {
|
|||||||
/// Tool execution result.
|
/// Tool execution result.
|
||||||
///
|
///
|
||||||
/// Every output has a mandatory `summary` (1-2 lines) that persists in
|
/// Every output has a mandatory `summary` (1-2 lines) that persists in
|
||||||
/// conversation history even after pruning. The optional `content` carries
|
/// conversation history even after pruning. Optional text and binary details are
|
||||||
/// full text details. `attachments` are request-local and are never serialized.
|
/// committed to history and may later be omitted only by normal ToolResult pruning.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ToolOutput {
|
pub struct ToolOutput {
|
||||||
/// Short summary (1-2 lines). Always remains in history.
|
/// Short summary (1-2 lines). Always remains in history.
|
||||||
@@ -142,8 +170,8 @@ pub struct ToolOutput {
|
|||||||
/// Detailed text output. Removed by Prune when old enough.
|
/// Detailed text output. Removed by Prune when old enough.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub content: Option<String>,
|
pub content: Option<String>,
|
||||||
/// Structured binary payloads for the immediately following model request.
|
/// Durable binary details handled by the same pruning lifecycle as `content`.
|
||||||
#[serde(skip, default)]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
pub attachments: Vec<Attachment>,
|
pub attachments: Vec<Attachment>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -409,8 +437,8 @@ pub struct ToolResult {
|
|||||||
/// Whether this is an error
|
/// Whether this is an error
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub is_error: bool,
|
pub is_error: bool,
|
||||||
/// Request-local structured payloads. Never serialized or persisted.
|
/// Durable binary details (prunable with `content`).
|
||||||
#[serde(skip, default)]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
pub attachments: Vec<Attachment>,
|
pub attachments: Vec<Attachment>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ edition.workspace = true
|
|||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
base64.workspace = true
|
||||||
llm-engine = { workspace = true }
|
llm-engine = { workspace = true }
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
|||||||
@@ -12,13 +12,36 @@
|
|||||||
//! `Reasoning::encrypted_content` is preserved because OpenAI Responses ZDR
|
//! `Reasoning::encrypted_content` is preserved because OpenAI Responses ZDR
|
||||||
//! requires it on stateless re-send.
|
//! requires it on stateless re-send.
|
||||||
|
|
||||||
use llm_engine::llm_client::types::{ContentPart, Item, Role};
|
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||||
use serde::{Deserialize, Serialize};
|
use llm_engine::{
|
||||||
|
llm_client::types::{ContentPart, Item, Role},
|
||||||
|
tool::{Attachment, ImageAttachment},
|
||||||
|
};
|
||||||
|
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
|
||||||
|
|
||||||
fn is_false(value: &bool) -> bool {
|
fn is_false(value: &bool) -> bool {
|
||||||
!*value
|
!*value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mod base64_bytes {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub fn serialize<S>(data: &[u8], serializer: S) -> Result<S::Ok, S::Error>
|
||||||
|
where
|
||||||
|
S: Serializer,
|
||||||
|
{
|
||||||
|
serializer.serialize_str(&STANDARD.encode(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let encoded = String::deserialize(deserializer)?;
|
||||||
|
STANDARD.decode(encoded).map_err(D::Error::custom)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
pub enum LoggedItem {
|
pub enum LoggedItem {
|
||||||
@@ -36,6 +59,8 @@ pub enum LoggedItem {
|
|||||||
summary: String,
|
summary: String,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
content: Option<String>,
|
content: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
attachments: Vec<LoggedAttachment>,
|
||||||
#[serde(default, skip_serializing_if = "is_false")]
|
#[serde(default, skip_serializing_if = "is_false")]
|
||||||
is_error: bool,
|
is_error: bool,
|
||||||
},
|
},
|
||||||
@@ -67,6 +92,16 @@ pub enum LoggedContentPart {
|
|||||||
Refusal { refusal: String },
|
Refusal { refusal: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
|
pub enum LoggedAttachment {
|
||||||
|
Image {
|
||||||
|
mime_type: String,
|
||||||
|
#[serde(with = "base64_bytes")]
|
||||||
|
data: Vec<u8>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Item ↔ LoggedItem
|
// Item ↔ LoggedItem
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -92,12 +127,14 @@ impl From<&Item> for LoggedItem {
|
|||||||
call_id,
|
call_id,
|
||||||
summary,
|
summary,
|
||||||
content,
|
content,
|
||||||
|
attachments,
|
||||||
is_error,
|
is_error,
|
||||||
..
|
..
|
||||||
} => Self::ToolResult {
|
} => Self::ToolResult {
|
||||||
call_id: call_id.clone(),
|
call_id: call_id.clone(),
|
||||||
summary: summary.clone(),
|
summary: summary.clone(),
|
||||||
content: content.clone(),
|
content: content.clone(),
|
||||||
|
attachments: attachments.iter().map(LoggedAttachment::from).collect(),
|
||||||
is_error: *is_error,
|
is_error: *is_error,
|
||||||
},
|
},
|
||||||
Item::Reasoning {
|
Item::Reasoning {
|
||||||
@@ -146,6 +183,7 @@ impl From<LoggedItem> for Item {
|
|||||||
call_id,
|
call_id,
|
||||||
summary,
|
summary,
|
||||||
content,
|
content,
|
||||||
|
attachments,
|
||||||
is_error,
|
is_error,
|
||||||
} => Item::ToolResult {
|
} => Item::ToolResult {
|
||||||
id: None,
|
id: None,
|
||||||
@@ -153,7 +191,7 @@ impl From<LoggedItem> for Item {
|
|||||||
summary,
|
summary,
|
||||||
content,
|
content,
|
||||||
is_error,
|
is_error,
|
||||||
attachments: Vec::new(),
|
attachments: attachments.into_iter().map(Attachment::from).collect(),
|
||||||
},
|
},
|
||||||
LoggedItem::Reasoning {
|
LoggedItem::Reasoning {
|
||||||
text,
|
text,
|
||||||
@@ -214,9 +252,6 @@ impl From<&ContentPart> for LoggedContentPart {
|
|||||||
fn from(part: &ContentPart) -> Self {
|
fn from(part: &ContentPart) -> Self {
|
||||||
match part {
|
match part {
|
||||||
ContentPart::Text { text } => Self::Text { text: text.clone() },
|
ContentPart::Text { text } => Self::Text { text: text.clone() },
|
||||||
ContentPart::Image { .. } => Self::Text {
|
|
||||||
text: part.as_text().to_string(),
|
|
||||||
},
|
|
||||||
ContentPart::Refusal { refusal } => Self::Refusal {
|
ContentPart::Refusal { refusal } => Self::Refusal {
|
||||||
refusal: refusal.clone(),
|
refusal: refusal.clone(),
|
||||||
},
|
},
|
||||||
@@ -233,6 +268,27 @@ impl From<LoggedContentPart> for ContentPart {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<&Attachment> for LoggedAttachment {
|
||||||
|
fn from(attachment: &Attachment) -> Self {
|
||||||
|
match attachment {
|
||||||
|
Attachment::Image(image) => Self::Image {
|
||||||
|
mime_type: image.mime_type().to_string(),
|
||||||
|
data: image.data().to_vec(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<LoggedAttachment> for Attachment {
|
||||||
|
fn from(attachment: LoggedAttachment) -> Self {
|
||||||
|
match attachment {
|
||||||
|
LoggedAttachment::Image { mime_type, data } => {
|
||||||
|
Self::Image(ImageAttachment::new(mime_type, data))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -375,7 +431,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn tool_result_persistence_drops_binary_attachments() {
|
fn tool_result_persistence_round_trips_binary_attachments() {
|
||||||
let original = Item::tool_result_item_with_attachments(
|
let original = Item::tool_result_item_with_attachments(
|
||||||
"call_image",
|
"call_image",
|
||||||
"attached",
|
"attached",
|
||||||
@@ -390,11 +446,17 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let logged: LoggedItem = (&original).into();
|
let logged: LoggedItem = (&original).into();
|
||||||
let json = serde_json::to_string(&logged).unwrap();
|
let json = serde_json::to_string(&logged).unwrap();
|
||||||
assert!(!json.contains("secret-image-body"));
|
assert!(json.contains("attachments"));
|
||||||
assert!(!json.contains("attachments"));
|
assert!(json.contains("c2VjcmV0LWltYWdlLWJvZHk="));
|
||||||
|
|
||||||
match Item::from(logged) {
|
let restored: LoggedItem = serde_json::from_str(&json).unwrap();
|
||||||
Item::ToolResult { attachments, .. } => assert!(attachments.is_empty()),
|
match Item::from(restored) {
|
||||||
|
Item::ToolResult { attachments, .. } => assert!(matches!(
|
||||||
|
attachments.as_slice(),
|
||||||
|
[Attachment::Image(image)]
|
||||||
|
if image.mime_type() == "image/png"
|
||||||
|
&& image.data() == b"secret-image-body"
|
||||||
|
)),
|
||||||
other => panic!("unexpected variant: {other:?}"),
|
other => panic!("unexpected variant: {other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -473,6 +473,36 @@ mod tests {
|
|||||||
assert!(state.history[2].is_tool_result());
|
assert!(state.history[2].is_tool_result());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replay_restores_durable_tool_image_detail() {
|
||||||
|
let entry = LogEntry::ToolResult {
|
||||||
|
ts: 3500,
|
||||||
|
item: Item::tool_result_item_with_attachments(
|
||||||
|
"call_image",
|
||||||
|
"attached",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
vec![llm_engine::tool::Attachment::Image(
|
||||||
|
llm_engine::tool::ImageAttachment::new("image/png", b"durable-image".to_vec()),
|
||||||
|
)],
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
|
};
|
||||||
|
let persisted = serde_json::to_string(&entry).unwrap();
|
||||||
|
let restored_entry: LogEntry = serde_json::from_str(&persisted).unwrap();
|
||||||
|
let state = collect_state(&[restored_entry]);
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
&state.history[0],
|
||||||
|
Item::ToolResult { attachments, .. }
|
||||||
|
if matches!(
|
||||||
|
attachments.as_slice(),
|
||||||
|
[llm_engine::tool::Attachment::Image(image)]
|
||||||
|
if image.data() == b"durable-image"
|
||||||
|
)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn replay_config_changed() {
|
fn replay_config_changed() {
|
||||||
let state = collect_state(&[
|
let state = collect_state(&[
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ fn meta_has_description_and_schema() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn view_image_reads_scoped_bytes_without_serializing_them_as_text() {
|
async fn view_image_reads_scoped_bytes_into_durable_tool_detail() {
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
let spill = TempDir::new().unwrap();
|
let spill = TempDir::new().unwrap();
|
||||||
let scope = scope_with_spill(dir.path(), spill.path());
|
let scope = scope_with_spill(dir.path(), spill.path());
|
||||||
@@ -119,7 +119,10 @@ async fn view_image_reads_scoped_bytes_without_serializing_them_as_text() {
|
|||||||
assert_eq!(image.data(), png);
|
assert_eq!(image.data(), png);
|
||||||
let serialized = serde_json::to_string(&output).unwrap();
|
let serialized = serde_json::to_string(&output).unwrap();
|
||||||
assert!(!serialized.contains("private-image-body"));
|
assert!(!serialized.contains("private-image-body"));
|
||||||
assert!(!serialized.contains("attachments"));
|
assert!(serialized.contains("attachments"));
|
||||||
|
let restored: llm_engine::tool::ToolOutput = serde_json::from_str(&serialized).unwrap();
|
||||||
|
let llm_engine::tool::Attachment::Image(restored_image) = &restored.attachments[0];
|
||||||
|
assert_eq!(restored_image.data(), png);
|
||||||
|
|
||||||
let escaped = call_err(&tool, json!({ "path": "../outside.png" })).await;
|
let escaped = call_err(&tool, json!({ "path": "../outside.png" })).await;
|
||||||
assert!(escaped.to_string().contains("scope") || escaped.to_string().contains("path"));
|
assert!(escaped.to_string().contains("scope") || escaped.to_string().contains("path"));
|
||||||
|
|||||||
@@ -271,9 +271,20 @@ impl SessionCapture {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
Item::ToolResult {
|
Item::ToolResult {
|
||||||
summary, content, ..
|
summary,
|
||||||
|
content,
|
||||||
|
attachments,
|
||||||
|
..
|
||||||
} => {
|
} => {
|
||||||
let text = format!("{summary}\n{}", content.as_deref().unwrap_or_default());
|
let attachment_marker = if attachments.is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("\n[{} image attachment(s)]", attachments.len())
|
||||||
|
};
|
||||||
|
let text = format!(
|
||||||
|
"{summary}\n{}{attachment_marker}",
|
||||||
|
content.as_deref().unwrap_or_default(),
|
||||||
|
);
|
||||||
index.push(ReferenceEntry {
|
index.push(ReferenceEntry {
|
||||||
id: SessionEntryRef::new(idx),
|
id: SessionEntryRef::new(idx),
|
||||||
entry_range,
|
entry_range,
|
||||||
@@ -506,21 +517,29 @@ fn render_item(
|
|||||||
Item::ToolResult {
|
Item::ToolResult {
|
||||||
summary,
|
summary,
|
||||||
content,
|
content,
|
||||||
|
attachments,
|
||||||
is_error,
|
is_error,
|
||||||
..
|
..
|
||||||
} => match detail {
|
} => {
|
||||||
|
let attachment_line = if attachments.is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("\nattachments: {} image(s)", attachments.len())
|
||||||
|
};
|
||||||
|
match detail {
|
||||||
ReadDetail::Compact => format!(
|
ReadDetail::Compact => format!(
|
||||||
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted)",
|
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted){attachment_line}",
|
||||||
entry.id,
|
|
||||||
if *is_error { " error" } else { "" }
|
|
||||||
),
|
|
||||||
ReadDetail::Full => format!(
|
|
||||||
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}",
|
|
||||||
entry.id,
|
entry.id,
|
||||||
if *is_error { " error" } else { "" },
|
if *is_error { " error" } else { "" },
|
||||||
content.as_deref().unwrap_or_default()
|
|
||||||
),
|
),
|
||||||
},
|
ReadDetail::Full => format!(
|
||||||
|
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}{attachment_line}",
|
||||||
|
entry.id,
|
||||||
|
if *is_error { " error" } else { "" },
|
||||||
|
content.as_deref().unwrap_or_default(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id),
|
Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id),
|
||||||
};
|
};
|
||||||
truncate_chars(&text, max_bytes)
|
truncate_chars(&text, max_bytes)
|
||||||
@@ -633,6 +652,38 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tool_image_projection_exposes_only_bounded_metadata() {
|
||||||
|
let view = SessionCapture::new(
|
||||||
|
"segment-1",
|
||||||
|
vec![Item::tool_result_item_with_attachments(
|
||||||
|
"c1",
|
||||||
|
"attached",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
vec![llm_engine::tool::Attachment::Image(
|
||||||
|
llm_engine::tool::ImageAttachment::new(
|
||||||
|
"image/png",
|
||||||
|
b"private-image-body".to_vec(),
|
||||||
|
),
|
||||||
|
)],
|
||||||
|
)],
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = view.read(
|
||||||
|
ReadSelector::Id("E00000000"),
|
||||||
|
ReadOptions {
|
||||||
|
include_tools: true,
|
||||||
|
detail: ReadDetail::Full,
|
||||||
|
..ReadOptions::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert_eq!(result.entries.len(), 1);
|
||||||
|
assert!(result.entries[0].text.contains("attachments: 1 image(s)"));
|
||||||
|
assert!(!result.entries[0].text.contains("private-image-body"));
|
||||||
|
assert!(!result.entries[0].text.contains("cHJpdmF0ZS"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn system_prompt_and_reasoning_are_excluded_from_every_projection() {
|
fn system_prompt_and_reasoning_are_excluded_from_every_projection() {
|
||||||
let view = SessionCapture::new(
|
let view = SessionCapture::new(
|
||||||
|
|||||||
@@ -6507,6 +6507,7 @@ mod build_summary_prompt_tests {
|
|||||||
call_id: "call-1".into(),
|
call_id: "call-1".into(),
|
||||||
summary: "wrote a file".into(),
|
summary: "wrote a file".into(),
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
is_error: false,
|
is_error: false,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -6548,6 +6549,7 @@ mod build_summary_prompt_tests {
|
|||||||
call_id: "call-1".into(),
|
call_id: "call-1".into(),
|
||||||
summary: "wrote a file".into(),
|
summary: "wrote a file".into(),
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
is_error: false,
|
is_error: false,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# Request-local image injection bypassed durable history and pruning
|
||||||
|
|
||||||
|
Date: 2026-08-06
|
||||||
|
|
||||||
|
## Symptom
|
||||||
|
|
||||||
|
The first `ViewImage` implementation stored image bytes in `ToolOutput.attachments` with
|
||||||
|
`#[serde(skip)]`, projected them into a synthetic user image only while building one provider
|
||||||
|
request, and then cleared the attachment from Worker history. The model could therefore answer
|
||||||
|
from pixels that disappeared from the next turn, session restore, pruning, and compaction
|
||||||
|
observation.
|
||||||
|
|
||||||
|
## Why this was wrong
|
||||||
|
|
||||||
|
Yoi's context policy requires new model-visible input to be appended and committed to
|
||||||
|
`worker.history` before request construction. Pure pruning may alter a request-context clone
|
||||||
|
because the projection is deterministic from durable history; request-local input injection is
|
||||||
|
not equivalent.
|
||||||
|
|
||||||
|
The transient image also changed the middle of the next prompt prefix without an explicit prune,
|
||||||
|
which reduced prompt-cache reuse and left later turns without the evidence behind the assistant
|
||||||
|
response.
|
||||||
|
|
||||||
|
Local reference review confirmed the intended pattern:
|
||||||
|
|
||||||
|
- Codex records `view_image` as image-bearing `FunctionCallOutput` content and explicitly tests
|
||||||
|
that no separate image message is injected.
|
||||||
|
- OpenCode persists image file parts in Session state and makes media removal an explicit
|
||||||
|
compaction/pruning decision.
|
||||||
|
|
||||||
|
## Resolution
|
||||||
|
|
||||||
|
Image attachments are now durable ToolResult detail:
|
||||||
|
|
||||||
|
- Image bytes are base64-serialized through normal Item/session-log persistence.
|
||||||
|
- OpenAI Responses lowers them to `function_call_output` content items.
|
||||||
|
- OpenAI Chat deterministically lowers the durable ToolResult to tool text followed by a user
|
||||||
|
image message while preserving parallel-tool-result ordering.
|
||||||
|
- The existing ToolResult pruning projection removes both text detail and image attachments from
|
||||||
|
the request-context clone while retaining the summary and original persistent history.
|
||||||
|
- Session exploration and compaction-facing projections expose only bounded attachment metadata,
|
||||||
|
never raw base64 as ordinary text.
|
||||||
|
|
||||||
|
## Guardrail
|
||||||
|
|
||||||
|
Do not use `#[serde(skip)]`, post-build clearing, or one-shot synthetic messages for input that can
|
||||||
|
influence model output. Provider-specific synthetic messages are acceptable only when they are a
|
||||||
|
stable projection of a committed history item and are regenerated identically until an explicit
|
||||||
|
pruning or compaction boundary.
|
||||||
Reference in New Issue
Block a user