feat: add scoped multimodal image attachments
This commit is contained in:
@@ -7,7 +7,7 @@ license.workspace = true
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
codex = ["dep:base64", "dep:chrono"]
|
codex = ["dep:chrono"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
@@ -21,7 +21,7 @@ tokio-util = "0.7"
|
|||||||
reqwest = { version = "0.13", default-features = false, features = ["stream", "json", "native-tls", "http2"] }
|
reqwest = { version = "0.13", default-features = false, features = ["stream", "json", "native-tls", "http2"] }
|
||||||
eventsource-stream = "0.2"
|
eventsource-stream = "0.2"
|
||||||
zstd = "0.13"
|
zstd = "0.13"
|
||||||
base64 = { version = "0.22.1", optional = true }
|
base64 = "0.22.1"
|
||||||
chrono = { version = "0.4", default-features = false, features = ["serde", "clock"], optional = true }
|
chrono = { version = "0.4", default-features = false, features = ["serde", "clock"], optional = true }
|
||||||
llm-engine-macros = { workspace = true }
|
llm-engine-macros = { workspace = true }
|
||||||
|
|
||||||
|
|||||||
@@ -19,15 +19,19 @@ use crate::{
|
|||||||
},
|
},
|
||||||
llm_client::{
|
llm_client::{
|
||||||
ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream,
|
ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream,
|
||||||
ToolDefinition, error::is_retryable, event::Event, retry::RetryPolicy,
|
ToolDefinition,
|
||||||
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT, types::parse_tool_arguments,
|
error::is_retryable,
|
||||||
|
event::Event,
|
||||||
|
retry::RetryPolicy,
|
||||||
|
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT,
|
||||||
|
types::{ContentPart, parse_tool_arguments},
|
||||||
},
|
},
|
||||||
state::{EngineState, Locked, Mutable},
|
state::{EngineState, Locked, Mutable},
|
||||||
timeline::event::{ErrorEvent, StatusEvent, UsageEvent},
|
timeline::event::{ErrorEvent, StatusEvent, UsageEvent},
|
||||||
timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector},
|
timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector},
|
||||||
tool::{
|
tool::{
|
||||||
ToolCall, ToolDefinition as EngineToolDefinition, ToolError, ToolExecutionContext,
|
Attachment, ToolCall, ToolDefinition as EngineToolDefinition, ToolError,
|
||||||
ToolOutputLimits, ToolResult, truncate_content,
|
ToolExecutionContext, ToolOutputLimits, ToolResult, truncate_content,
|
||||||
},
|
},
|
||||||
tool_server::{ToolServer, ToolServerHandle},
|
tool_server::{ToolServer, ToolServerHandle},
|
||||||
};
|
};
|
||||||
@@ -155,6 +159,33 @@ enum StreamCompletion {
|
|||||||
Interrupted { reason: String },
|
Interrupted { reason: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn project_transient_attachments(items: &mut Vec<Item>) {
|
||||||
|
let mut projected = Vec::with_capacity(items.len());
|
||||||
|
let mut pending_parts = Vec::new();
|
||||||
|
|
||||||
|
for mut item in items.drain(..) {
|
||||||
|
let is_tool_result = matches!(item, Item::ToolResult { .. });
|
||||||
|
if !is_tool_result && !pending_parts.is_empty() {
|
||||||
|
projected.push(Item::user_message_parts(std::mem::take(&mut pending_parts)));
|
||||||
|
}
|
||||||
|
if let Item::ToolResult { attachments, .. } = &item {
|
||||||
|
for attachment in attachments {
|
||||||
|
let Attachment::Image(image) = attachment;
|
||||||
|
pending_parts.push(ContentPart::image(
|
||||||
|
image.mime_type(),
|
||||||
|
Arc::from(image.data()),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
item.clear_transient_attachments();
|
||||||
|
projected.push(item);
|
||||||
|
}
|
||||||
|
if !pending_parts.is_empty() {
|
||||||
|
projected.push(Item::user_message_parts(pending_parts));
|
||||||
|
}
|
||||||
|
*items = projected;
|
||||||
|
}
|
||||||
|
|
||||||
pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
|
pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
|
||||||
/// LLM client
|
/// LLM client
|
||||||
client: C,
|
client: C,
|
||||||
@@ -1249,6 +1280,8 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
cb(current_llm_call);
|
cb(current_llm_call);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
project_transient_attachments(&mut request_context);
|
||||||
|
|
||||||
// Stream LLM response
|
// Stream LLM response
|
||||||
self.emit_lifecycle_trace(
|
self.emit_lifecycle_trace(
|
||||||
current_turn,
|
current_turn,
|
||||||
@@ -1257,6 +1290,12 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
items_trace_payload(&request_context, tool_definitions.len(), None, false),
|
items_trace_payload(&request_context, tool_definitions.len(), None, false),
|
||||||
);
|
);
|
||||||
let request = self.build_request(&tool_definitions, &request_context);
|
let request = self.build_request(&tool_definitions, &request_context);
|
||||||
|
// Structured attachments are single-use provider payloads. `request`
|
||||||
|
// owns its Arc-backed copy; history is cleared before any later turn
|
||||||
|
// can persist or resend the binary body.
|
||||||
|
for item in &mut self.history {
|
||||||
|
item.clear_transient_attachments();
|
||||||
|
}
|
||||||
self.emit_lifecycle_trace(
|
self.emit_lifecycle_trace(
|
||||||
current_turn,
|
current_turn,
|
||||||
current_llm_call,
|
current_llm_call,
|
||||||
@@ -1618,11 +1657,12 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
|
|||||||
// Route per-result pushes through the callback path so
|
// Route per-result pushes through the callback path so
|
||||||
// observers see each tool result as it lands.
|
// observers see each tool result as it lands.
|
||||||
let items = results.into_iter().map(|result| {
|
let items = results.into_iter().map(|result| {
|
||||||
Item::tool_result_item(
|
Item::tool_result_item_with_attachments(
|
||||||
&result.tool_use_id,
|
&result.tool_use_id,
|
||||||
&result.summary,
|
&result.summary,
|
||||||
result.content,
|
result.content,
|
||||||
result.is_error,
|
result.is_error,
|
||||||
|
result.attachments,
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
self.append_history_items(items)?;
|
self.append_history_items(items)?;
|
||||||
@@ -2159,8 +2199,50 @@ fn item_kind(item: &Item) -> &'static str {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::llm_client::types::Role;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn transient_tool_attachment_becomes_non_persistent_user_image_part() {
|
||||||
|
let body: Arc<[u8]> = Arc::from(&b"secret-image-body"[..]);
|
||||||
|
let mut items = vec![Item::tool_result_item_with_attachments(
|
||||||
|
"call_image",
|
||||||
|
"attached",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
vec![Attachment::Image(crate::tool::ImageAttachment::new(
|
||||||
|
"image/png",
|
||||||
|
body.clone(),
|
||||||
|
))],
|
||||||
|
)];
|
||||||
|
|
||||||
|
project_transient_attachments(&mut items);
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
&items[0],
|
||||||
|
Item::ToolResult { attachments, .. } if attachments.is_empty()
|
||||||
|
));
|
||||||
|
match &items[1] {
|
||||||
|
Item::Message {
|
||||||
|
role: Role::User,
|
||||||
|
content,
|
||||||
|
..
|
||||||
|
} => match &content[0] {
|
||||||
|
ContentPart::Image { media_type, source } => {
|
||||||
|
assert_eq!(media_type, "image/png");
|
||||||
|
assert_eq!(source.data(), body.as_ref());
|
||||||
|
assert_eq!(source.bytes(), body.len());
|
||||||
|
assert_eq!(content[0].as_text(), "[image attachment omitted]");
|
||||||
|
}
|
||||||
|
other => panic!("unexpected content part: {other:?}"),
|
||||||
|
},
|
||||||
|
other => panic!("unexpected projected item: {other:?}"),
|
||||||
|
}
|
||||||
|
let persisted = serde_json::to_string(&items).unwrap();
|
||||||
|
assert!(!persisted.contains("secret-image-body"));
|
||||||
|
assert!(!persisted.contains("base64"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn first_stream_event_timeout_returns_retryable_timeout() {
|
async fn first_stream_event_timeout_returns_retryable_timeout() {
|
||||||
let stream: ResponseStream = Box::pin(futures::stream::pending());
|
let stream: ResponseStream = Box::pin(futures::stream::pending());
|
||||||
|
|||||||
@@ -268,6 +268,9 @@ impl AnthropicScheme {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|p| match p {
|
.map(|p| match p {
|
||||||
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
|
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
|
||||||
|
ContentPart::Image { .. } => {
|
||||||
|
AnthropicContentPart::text(p.as_text().to_string())
|
||||||
|
}
|
||||||
ContentPart::Refusal { refusal } => {
|
ContentPart::Refusal { refusal } => {
|
||||||
AnthropicContentPart::text(refusal.clone())
|
AnthropicContentPart::text(refusal.clone())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ use serde_json::Value;
|
|||||||
use crate::llm_client::{
|
use crate::llm_client::{
|
||||||
Request,
|
Request,
|
||||||
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
||||||
types::{Item, Role, ToolDefinition, parse_tool_arguments},
|
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::OpenAIScheme;
|
use super::OpenAIScheme;
|
||||||
@@ -134,7 +134,7 @@ impl OpenAIScheme {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Convert items to messages
|
// Convert items to messages
|
||||||
messages.extend(self.convert_items_to_messages(&request.items));
|
messages.extend(self.convert_items_to_messages(&request.items, capability.vision));
|
||||||
|
|
||||||
let tools = request.tools.iter().map(|t| self.convert_tool(t)).collect();
|
let tools = request.tools.iter().map(|t| self.convert_tool(t)).collect();
|
||||||
|
|
||||||
@@ -185,7 +185,11 @@ impl OpenAIScheme {
|
|||||||
/// - Assistant messages have role "assistant"
|
/// - Assistant messages have role "assistant"
|
||||||
/// - Tool calls are within assistant messages as tool_calls array
|
/// - Tool calls are within assistant messages as tool_calls array
|
||||||
/// - Tool results have role "tool" with tool_call_id
|
/// - Tool results have role "tool" with tool_call_id
|
||||||
fn convert_items_to_messages(&self, items: &[Item]) -> Vec<OpenAIMessage> {
|
fn convert_items_to_messages(
|
||||||
|
&self,
|
||||||
|
items: &[Item],
|
||||||
|
supports_images: bool,
|
||||||
|
) -> Vec<OpenAIMessage> {
|
||||||
let mut messages = Vec::new();
|
let mut messages = Vec::new();
|
||||||
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
|
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
|
||||||
let mut pending_assistant_text: Option<String> = None;
|
let mut pending_assistant_text: Option<String> = None;
|
||||||
@@ -205,16 +209,45 @@ impl OpenAIScheme {
|
|||||||
Role::Assistant => "assistant",
|
Role::Assistant => "assistant",
|
||||||
Role::System => "system",
|
Role::System => "system",
|
||||||
};
|
};
|
||||||
|
let has_image = matches!(role, Role::User)
|
||||||
let text_content: String = content
|
&& supports_images
|
||||||
|
&& content
|
||||||
.iter()
|
.iter()
|
||||||
.map(|p| p.as_text())
|
.any(|part| matches!(part, ContentPart::Image { .. }));
|
||||||
|
let message_content = if has_image {
|
||||||
|
OpenAIContent::Parts(
|
||||||
|
content
|
||||||
|
.iter()
|
||||||
|
.map(|part| match part {
|
||||||
|
ContentPart::Text { text } => {
|
||||||
|
OpenAIContentPart::Text { text: text.clone() }
|
||||||
|
}
|
||||||
|
ContentPart::Image { media_type, source } => {
|
||||||
|
OpenAIContentPart::ImageUrl {
|
||||||
|
image_url: ImageUrl {
|
||||||
|
url: image_data_url(media_type, source.data()),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ContentPart::Refusal { refusal } => OpenAIContentPart::Text {
|
||||||
|
text: refusal.clone(),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
OpenAIContent::Text(
|
||||||
|
content
|
||||||
|
.iter()
|
||||||
|
.map(ContentPart::as_text)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join("");
|
.join(""),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
messages.push(OpenAIMessage {
|
messages.push(OpenAIMessage {
|
||||||
role: openai_role.to_string(),
|
role: openai_role.to_string(),
|
||||||
content: Some(OpenAIContent::Text(text_content)),
|
content: Some(message_content),
|
||||||
tool_calls: vec![],
|
tool_calls: vec![],
|
||||||
tool_call_id: None,
|
tool_call_id: None,
|
||||||
name: None,
|
name: None,
|
||||||
@@ -334,6 +367,13 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn vision_cap() -> ModelCapability {
|
||||||
|
ModelCapability {
|
||||||
|
vision: true,
|
||||||
|
..cap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_build_simple_request() {
|
fn test_build_simple_request() {
|
||||||
let scheme = OpenAIScheme::new();
|
let scheme = OpenAIScheme::new();
|
||||||
@@ -439,4 +479,92 @@ mod tests {
|
|||||||
assert_eq!(body.messages[1].tool_calls.len(), 1);
|
assert_eq!(body.messages[1].tool_calls.len(), 1);
|
||||||
assert_eq!(body.messages[2].role, "tool");
|
assert_eq!(body.messages[2].role, "tool");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parallel_tool_results_precede_synthetic_image_message() {
|
||||||
|
let scheme = OpenAIScheme::new();
|
||||||
|
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
||||||
|
let request = Request::new()
|
||||||
|
.item(Item::tool_call("call_image", "ViewImage", "{}"))
|
||||||
|
.item(Item::tool_call("call_text", "Read", "{}"))
|
||||||
|
.item(Item::tool_result_item(
|
||||||
|
"call_image",
|
||||||
|
"Attached image",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
))
|
||||||
|
.item(Item::tool_result_item(
|
||||||
|
"call_text",
|
||||||
|
"Read text",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
))
|
||||||
|
.item(Item::user_message_parts(vec![ContentPart::image(
|
||||||
|
"image/png",
|
||||||
|
image,
|
||||||
|
)]));
|
||||||
|
let json = serde_json::to_value(
|
||||||
|
&scheme
|
||||||
|
.build_request("gpt-4o", &request, &vision_cap())
|
||||||
|
.messages,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(json[0]["role"], "assistant");
|
||||||
|
assert_eq!(json[1]["role"], "tool");
|
||||||
|
assert_eq!(json[2]["role"], "tool");
|
||||||
|
assert_eq!(json[3]["role"], "user");
|
||||||
|
assert_eq!(json[3]["content"][0]["type"], "image_url");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tool_image_is_structured_as_following_user_content_without_persisting_bytes() {
|
||||||
|
let scheme = OpenAIScheme::new();
|
||||||
|
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
||||||
|
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
|
||||||
|
"image/png",
|
||||||
|
image.clone(),
|
||||||
|
));
|
||||||
|
let item = Item::tool_result_item_with_attachments(
|
||||||
|
"call_image",
|
||||||
|
"Attached image",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
vec![attachment],
|
||||||
|
);
|
||||||
|
let persisted = serde_json::to_string(&item).unwrap();
|
||||||
|
assert!(!persisted.contains("base64"));
|
||||||
|
assert!(!persisted.contains("attachments"));
|
||||||
|
|
||||||
|
let request = Request::new()
|
||||||
|
.item(Item::tool_call(
|
||||||
|
"call_image",
|
||||||
|
"ViewImage",
|
||||||
|
r#"{"path":"a.png"}"#,
|
||||||
|
))
|
||||||
|
.item(item)
|
||||||
|
.item(Item::user_message_parts(vec![ContentPart::image(
|
||||||
|
"image/png",
|
||||||
|
image,
|
||||||
|
)]));
|
||||||
|
let body = scheme.build_request("gpt-4o", &request, &vision_cap());
|
||||||
|
let json = serde_json::to_value(&body.messages).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(json[0]["role"], "assistant");
|
||||||
|
assert_eq!(json[1]["role"], "tool");
|
||||||
|
assert_eq!(json[2]["role"], "user");
|
||||||
|
assert_eq!(json[2]["content"][0]["type"], "image_url");
|
||||||
|
assert!(
|
||||||
|
json[2]["content"][0]["image_url"]["url"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.starts_with("data:image/png;base64,")
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut no_vision = cap();
|
||||||
|
no_vision.vision = false;
|
||||||
|
let disabled =
|
||||||
|
serde_json::to_string(&scheme.build_request("gpt-4o", &request, &no_vision)).unwrap();
|
||||||
|
assert!(!disabled.contains("data:image"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use serde_json::Value;
|
|||||||
use crate::llm_client::{
|
use crate::llm_client::{
|
||||||
Request,
|
Request,
|
||||||
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
|
||||||
types::{ContentPart, Item, Role, ToolDefinition, parse_tool_arguments},
|
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::OpenAIResponsesScheme;
|
use super::OpenAIResponsesScheme;
|
||||||
@@ -89,12 +89,7 @@ pub(crate) enum InputItem {
|
|||||||
arguments: String,
|
arguments: String,
|
||||||
},
|
},
|
||||||
/// function tool の結果(user 側)。
|
/// function tool の結果(user 側)。
|
||||||
FunctionCallOutput {
|
FunctionCallOutput { call_id: String, output: String },
|
||||||
call_id: String,
|
|
||||||
/// Responses は文字列 or 構造化 output を許すが、ここでは
|
|
||||||
/// `summary` + `content` を改行連結した文字列で送る。
|
|
||||||
output: String,
|
|
||||||
},
|
|
||||||
/// reasoning item。`encrypted_content` があれば必ず添える。
|
/// reasoning item。`encrypted_content` があれば必ず添える。
|
||||||
Reasoning {
|
Reasoning {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
@@ -118,6 +113,11 @@ pub(crate) enum InputItem {
|
|||||||
pub(crate) enum InputContent {
|
pub(crate) enum InputContent {
|
||||||
/// user / developer 側のテキスト
|
/// user / developer 側のテキスト
|
||||||
InputText { text: String },
|
InputText { text: String },
|
||||||
|
/// user 側の画像
|
||||||
|
InputImage {
|
||||||
|
image_url: String,
|
||||||
|
detail: &'static str,
|
||||||
|
},
|
||||||
/// assistant 側のテキスト
|
/// assistant 側のテキスト
|
||||||
OutputText { text: String },
|
OutputText { text: String },
|
||||||
}
|
}
|
||||||
@@ -173,7 +173,7 @@ impl OpenAIResponsesScheme {
|
|||||||
request: &Request,
|
request: &Request,
|
||||||
capability: &ModelCapability,
|
capability: &ModelCapability,
|
||||||
) -> ResponsesRequest {
|
) -> ResponsesRequest {
|
||||||
let input = convert_items_to_input(&request.items);
|
let input = convert_items_to_input(&request.items, capability.vision);
|
||||||
let tools = request.tools.iter().map(convert_tool).collect();
|
let tools = request.tools.iter().map(convert_tool).collect();
|
||||||
|
|
||||||
// Reasoning 投影: capability が Effort / Both をサポートし、かつ
|
// Reasoning 投影: capability が Effort / Both をサポートし、かつ
|
||||||
@@ -234,7 +234,7 @@ impl OpenAIResponsesScheme {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// `Item` 列を `input[]` に変換する。
|
/// `Item` 列を `input[]` に変換する。
|
||||||
fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
|
fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputItem> {
|
||||||
let mut out = Vec::with_capacity(items.len());
|
let mut out = Vec::with_capacity(items.len());
|
||||||
for item in items {
|
for item in items {
|
||||||
match item {
|
match item {
|
||||||
@@ -247,8 +247,17 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
|
|||||||
};
|
};
|
||||||
let parts: Vec<InputContent> = content
|
let parts: Vec<InputContent> = content
|
||||||
.iter()
|
.iter()
|
||||||
.map(|p| match p {
|
.map(|part| match part {
|
||||||
ContentPart::Text { text } => text_variant(text.clone()),
|
ContentPart::Text { text } => text_variant(text.clone()),
|
||||||
|
ContentPart::Image { media_type, source }
|
||||||
|
if matches!(role, Role::User) && supports_images =>
|
||||||
|
{
|
||||||
|
InputContent::InputImage {
|
||||||
|
image_url: image_data_url(media_type, source.data()),
|
||||||
|
detail: "auto",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ContentPart::Image { .. } => text_variant(part.as_text().to_string()),
|
||||||
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
|
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -277,13 +286,13 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
|
|||||||
content,
|
content,
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
let text = match content {
|
let output = match content {
|
||||||
Some(c) => format!("{summary}\n{c}"),
|
Some(c) => format!("{summary}\n{c}"),
|
||||||
None => summary.clone(),
|
None => summary.clone(),
|
||||||
};
|
};
|
||||||
out.push(InputItem::FunctionCallOutput {
|
out.push(InputItem::FunctionCallOutput {
|
||||||
call_id: call_id.clone(),
|
call_id: call_id.clone(),
|
||||||
output: text,
|
output,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Item::Reasoning {
|
Item::Reasoning {
|
||||||
@@ -690,4 +699,48 @@ mod tests {
|
|||||||
assert_eq!(json["tools"][0]["type"], "function");
|
assert_eq!(json["tools"][0]["type"], "function");
|
||||||
assert_eq!(json["tools"][0]["name"], "t");
|
assert_eq!(json["tools"][0]["name"], "t");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn synthetic_user_image_uses_responses_input_image_content() {
|
||||||
|
let scheme = OpenAIResponsesScheme::new();
|
||||||
|
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
|
||||||
|
let item = Item::tool_result_item_with_attachments(
|
||||||
|
"call_image",
|
||||||
|
"Attached image",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
vec![crate::tool::Attachment::Image(
|
||||||
|
crate::tool::ImageAttachment::new("image/png", image.clone()),
|
||||||
|
)],
|
||||||
|
);
|
||||||
|
let req = Request::new()
|
||||||
|
.item(Item::tool_call(
|
||||||
|
"call_image",
|
||||||
|
"ViewImage",
|
||||||
|
r#"{"path":"a.png"}"#,
|
||||||
|
))
|
||||||
|
.item(item)
|
||||||
|
.item(Item::user_message_parts(vec![ContentPart::image(
|
||||||
|
"image/png",
|
||||||
|
image,
|
||||||
|
)]));
|
||||||
|
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
|
||||||
|
let json = serde_json::to_value(&body).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(json["input"][1]["type"], "function_call_output");
|
||||||
|
assert_eq!(json["input"][2]["type"], "message");
|
||||||
|
assert_eq!(json["input"][2]["content"][0]["type"], "input_image");
|
||||||
|
assert!(
|
||||||
|
json["input"][2]["content"][0]["image_url"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.starts_with("data:image/png;base64,")
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut no_vision = cap_with_reasoning();
|
||||||
|
no_vision.vision = false;
|
||||||
|
let disabled =
|
||||||
|
serde_json::to_string(&scheme.build_request("gpt-5", &req, &no_vision)).unwrap();
|
||||||
|
assert!(!disabled.contains("data:image"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,12 +9,19 @@
|
|||||||
|
|
||||||
use std::{fmt, sync::Arc};
|
use std::{fmt, sync::Arc};
|
||||||
|
|
||||||
|
use crate::tool::Attachment;
|
||||||
|
use base64::Engine as _;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
fn is_false(value: &bool) -> bool {
|
fn is_false(value: &bool) -> bool {
|
||||||
!*value
|
!*value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn image_data_url(media_type: &str, data: &[u8]) -> String {
|
||||||
|
let encoded = base64::engine::general_purpose::STANDARD.encode(data);
|
||||||
|
format!("data:{media_type};base64,{encoded}")
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Item - The core unit of conversation
|
// Item - The core unit of conversation
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -117,6 +124,9 @@ pub enum Item {
|
|||||||
/// Whether the tool result represents an execution error.
|
/// Whether the tool result represents an execution error.
|
||||||
#[serde(default, skip_serializing_if = "is_false")]
|
#[serde(default, skip_serializing_if = "is_false")]
|
||||||
is_error: bool,
|
is_error: bool,
|
||||||
|
/// Request-local structured payloads. Never serialized or persisted.
|
||||||
|
#[serde(skip, default)]
|
||||||
|
attachments: Vec<Attachment>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Reasoning/thinking item
|
/// Reasoning/thinking item
|
||||||
@@ -250,6 +260,17 @@ impl Item {
|
|||||||
summary: impl Into<String>,
|
summary: impl Into<String>,
|
||||||
content: Option<String>,
|
content: Option<String>,
|
||||||
is_error: bool,
|
is_error: bool,
|
||||||
|
) -> Self {
|
||||||
|
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a tool result item with request-local structured attachments.
|
||||||
|
pub fn tool_result_item_with_attachments(
|
||||||
|
call_id: impl Into<String>,
|
||||||
|
summary: impl Into<String>,
|
||||||
|
content: Option<String>,
|
||||||
|
is_error: bool,
|
||||||
|
attachments: Vec<Attachment>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self::ToolResult {
|
Self::ToolResult {
|
||||||
id: None,
|
id: None,
|
||||||
@@ -257,6 +278,14 @@ impl Item {
|
|||||||
summary: summary.into(),
|
summary: summary.into(),
|
||||||
content,
|
content,
|
||||||
is_error,
|
is_error,
|
||||||
|
attachments,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop request-local attachments after constructing the provider request.
|
||||||
|
pub fn clear_transient_attachments(&mut self) {
|
||||||
|
if let Self::ToolResult { attachments, .. } = self {
|
||||||
|
attachments.clear();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -428,6 +457,38 @@ pub fn parse_tool_arguments(arguments: &str) -> serde_json::Value {
|
|||||||
// Content Parts - Components within message items
|
// Content Parts - Components within message items
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct ImageSource {
|
||||||
|
bytes: usize,
|
||||||
|
#[serde(skip, default)]
|
||||||
|
data: Arc<[u8]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ImageSource {
|
||||||
|
pub fn new(data: Arc<[u8]>) -> Self {
|
||||||
|
Self {
|
||||||
|
bytes: data.len(),
|
||||||
|
data,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn data(&self) -> &[u8] {
|
||||||
|
&self.data
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn bytes(&self) -> usize {
|
||||||
|
self.bytes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for ImageSource {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
f.debug_struct("ImageSource")
|
||||||
|
.field("bytes", &self.bytes)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Content part within a message item
|
/// Content part within a message item
|
||||||
///
|
///
|
||||||
/// Text content is role-agnostic; the containing Item's Role determines
|
/// Text content is role-agnostic; the containing Item's Role determines
|
||||||
@@ -441,6 +502,12 @@ pub enum ContentPart {
|
|||||||
text: String,
|
text: String,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// Request-local image content. The source bytes are never serialized.
|
||||||
|
Image {
|
||||||
|
media_type: String,
|
||||||
|
source: ImageSource,
|
||||||
|
},
|
||||||
|
|
||||||
/// Refusal content (for assistant messages)
|
/// Refusal content (for assistant messages)
|
||||||
Refusal {
|
Refusal {
|
||||||
/// The refusal message
|
/// The refusal message
|
||||||
@@ -461,10 +528,20 @@ impl ContentPart {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the text content regardless of type
|
pub fn image(media_type: impl Into<String>, data: Arc<[u8]>) -> Self {
|
||||||
|
Self::Image {
|
||||||
|
media_type: media_type.into(),
|
||||||
|
source: ImageSource::new(data),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a bounded textual projection. Image content is represented by an
|
||||||
|
/// explicit placeholder rather than silently becoming an empty string,
|
||||||
|
/// filesystem path, data URL, or base64 body.
|
||||||
pub fn as_text(&self) -> &str {
|
pub fn as_text(&self) -> &str {
|
||||||
match self {
|
match self {
|
||||||
Self::Text { text } => text,
|
Self::Text { text } => text,
|
||||||
|
Self::Image { .. } => "[image attachment omitted]",
|
||||||
Self::Refusal { refusal } => refusal,
|
Self::Refusal { refusal } => refusal,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,7 @@
|
|||||||
//! Traits for defining tools callable by LLM.
|
//! Traits for defining tools callable by LLM.
|
||||||
//! Usually auto-implemented using the `#[tool]` macro.
|
//! Usually auto-implemented using the `#[tool]` macro.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::{collections::HashMap, fmt, sync::Arc};
|
||||||
use std::sync::Arc;
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -89,19 +88,63 @@ pub(crate) fn truncate_content(content: &mut String, limit: usize) {
|
|||||||
content.push_str(&suffix_template.replace("%BYTES%", &dropped.to_string()));
|
content.push_str(&suffix_template.replace("%BYTES%", &dropped.to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
|
pub struct ImageAttachment {
|
||||||
|
mime_type: String,
|
||||||
|
data: Arc<[u8]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ImageAttachment {
|
||||||
|
pub fn new(mime_type: impl Into<String>, data: impl Into<Arc<[u8]>>) -> Self {
|
||||||
|
Self {
|
||||||
|
mime_type: mime_type.into(),
|
||||||
|
data: data.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn mime_type(&self) -> &str {
|
||||||
|
&self.mime_type
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn data(&self) -> &[u8] {
|
||||||
|
&self.data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for ImageAttachment {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
f.debug_struct("ImageAttachment")
|
||||||
|
.field("mime_type", &self.mime_type)
|
||||||
|
.field("bytes", &self.data.len())
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Request-local binary payload emitted by a tool.
|
||||||
|
///
|
||||||
|
/// Attachments are deliberately excluded from serde. They may be projected into
|
||||||
|
/// the immediately following provider request, but never into persisted history,
|
||||||
|
/// protocol events, logs, or telemetry.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Attachment {
|
||||||
|
Image(ImageAttachment),
|
||||||
|
}
|
||||||
|
|
||||||
/// Tool execution result.
|
/// Tool execution result.
|
||||||
///
|
///
|
||||||
/// Every output has a mandatory `summary` (1-2 lines) that persists in
|
/// Every output has a mandatory `summary` (1-2 lines) that persists in
|
||||||
/// conversation history even after pruning. The optional `content` carries
|
/// conversation history even after pruning. The optional `content` carries
|
||||||
/// full details and is removed by the Prune mechanism when the context
|
/// full text details. `attachments` are request-local and are never serialized.
|
||||||
/// grows too large.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ToolOutput {
|
pub struct ToolOutput {
|
||||||
/// Short summary (1-2 lines). Always remains in history.
|
/// Short summary (1-2 lines). Always remains in history.
|
||||||
pub summary: String,
|
pub summary: String,
|
||||||
/// Detailed output. Removed by Prune when old enough.
|
/// Detailed text output. Removed by Prune when old enough.
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub content: Option<String>,
|
pub content: Option<String>,
|
||||||
|
/// Structured binary payloads for the immediately following model request.
|
||||||
|
#[serde(skip, default)]
|
||||||
|
pub attachments: Vec<Attachment>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<String> for ToolOutput {
|
impl From<String> for ToolOutput {
|
||||||
@@ -110,6 +153,7 @@ impl From<String> for ToolOutput {
|
|||||||
ToolOutput {
|
ToolOutput {
|
||||||
summary: s,
|
summary: s,
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
let lines = s.lines().count();
|
let lines = s.lines().count();
|
||||||
@@ -118,6 +162,7 @@ impl From<String> for ToolOutput {
|
|||||||
ToolOutput {
|
ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(s),
|
content: Some(s),
|
||||||
|
attachments: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -364,6 +409,9 @@ pub struct ToolResult {
|
|||||||
/// Whether this is an error
|
/// Whether this is an error
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub is_error: bool,
|
pub is_error: bool,
|
||||||
|
/// Request-local structured payloads. Never serialized or persisted.
|
||||||
|
#[serde(skip, default)]
|
||||||
|
pub attachments: Vec<Attachment>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ToolResult {
|
impl ToolResult {
|
||||||
@@ -374,6 +422,7 @@ impl ToolResult {
|
|||||||
summary: output.summary,
|
summary: output.summary,
|
||||||
content: output.content,
|
content: output.content,
|
||||||
is_error: false,
|
is_error: false,
|
||||||
|
attachments: output.attachments,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -384,6 +433,7 @@ impl ToolResult {
|
|||||||
summary: message.into(),
|
summary: message.into(),
|
||||||
content: None,
|
content: None,
|
||||||
is_error: true,
|
is_error: true,
|
||||||
|
attachments: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,6 +83,8 @@ pub struct FeatureConfigPartial {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub web: Option<FeatureFlagConfigPartial>,
|
pub web: Option<FeatureFlagConfigPartial>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
pub image: Option<FeatureFlagConfigPartial>,
|
||||||
|
#[serde(default)]
|
||||||
pub sub_worker: Option<FeatureFlagConfigPartial>,
|
pub sub_worker: Option<FeatureFlagConfigPartial>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub flow: Option<FeatureFlagConfigPartial>,
|
pub flow: Option<FeatureFlagConfigPartial>,
|
||||||
@@ -104,6 +106,7 @@ impl FeatureConfigPartial {
|
|||||||
task: merge_option(self.task, other.task, FeatureFlagConfigPartial::merge),
|
task: merge_option(self.task, other.task, FeatureFlagConfigPartial::merge),
|
||||||
memory: merge_option(self.memory, other.memory, MemoryFeatureConfigPartial::merge),
|
memory: merge_option(self.memory, other.memory, MemoryFeatureConfigPartial::merge),
|
||||||
web: merge_option(self.web, other.web, FeatureFlagConfigPartial::merge),
|
web: merge_option(self.web, other.web, FeatureFlagConfigPartial::merge),
|
||||||
|
image: merge_option(self.image, other.image, FeatureFlagConfigPartial::merge),
|
||||||
sub_worker: merge_option(
|
sub_worker: merge_option(
|
||||||
self.sub_worker,
|
self.sub_worker,
|
||||||
other.sub_worker,
|
other.sub_worker,
|
||||||
@@ -189,6 +192,7 @@ impl From<FeatureConfigPartial> for FeatureConfig {
|
|||||||
.map(MemoryFeatureConfig::from)
|
.map(MemoryFeatureConfig::from)
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
web: value.web.map(FeatureFlagConfig::from).unwrap_or_default(),
|
web: value.web.map(FeatureFlagConfig::from).unwrap_or_default(),
|
||||||
|
image: value.image.map(FeatureFlagConfig::from).unwrap_or_default(),
|
||||||
sub_worker: value
|
sub_worker: value
|
||||||
.sub_worker
|
.sub_worker
|
||||||
.map(FeatureFlagConfig::from)
|
.map(FeatureFlagConfig::from)
|
||||||
@@ -282,6 +286,7 @@ impl From<FeatureConfig> for FeatureConfigPartial {
|
|||||||
task: Some(value.task.into()),
|
task: Some(value.task.into()),
|
||||||
memory: Some(value.memory.into()),
|
memory: Some(value.memory.into()),
|
||||||
web: Some(value.web.into()),
|
web: Some(value.web.into()),
|
||||||
|
image: Some(value.image.into()),
|
||||||
sub_worker: Some(value.sub_worker.into()),
|
sub_worker: Some(value.sub_worker.into()),
|
||||||
flow: Some(value.flow.into()),
|
flow: Some(value.flow.into()),
|
||||||
worker: Some(value.worker.into()),
|
worker: Some(value.worker.into()),
|
||||||
|
|||||||
@@ -111,6 +111,8 @@ pub struct FeatureConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub web: FeatureFlagConfig,
|
pub web: FeatureFlagConfig,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
pub image: FeatureFlagConfig,
|
||||||
|
#[serde(default)]
|
||||||
pub sub_worker: FeatureFlagConfig,
|
pub sub_worker: FeatureFlagConfig,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub flow: FeatureFlagConfig,
|
pub flow: FeatureFlagConfig,
|
||||||
@@ -132,6 +134,7 @@ impl Default for FeatureConfig {
|
|||||||
task: FeatureFlagConfig::disabled(),
|
task: FeatureFlagConfig::disabled(),
|
||||||
memory: MemoryFeatureConfig::disabled(),
|
memory: MemoryFeatureConfig::disabled(),
|
||||||
web: FeatureFlagConfig::disabled(),
|
web: FeatureFlagConfig::disabled(),
|
||||||
|
image: FeatureFlagConfig::disabled(),
|
||||||
sub_worker: FeatureFlagConfig::disabled(),
|
sub_worker: FeatureFlagConfig::disabled(),
|
||||||
flow: FeatureFlagConfig::disabled(),
|
flow: FeatureFlagConfig::disabled(),
|
||||||
worker: FeatureFlagConfig::disabled(),
|
worker: FeatureFlagConfig::disabled(),
|
||||||
|
|||||||
@@ -962,6 +962,7 @@ fn builtin_base_profile_artifact() -> serde_json::Value {
|
|||||||
"task": { "enabled": true },
|
"task": { "enabled": true },
|
||||||
"memory": { "enabled": true },
|
"memory": { "enabled": true },
|
||||||
"web": { "enabled": true },
|
"web": { "enabled": true },
|
||||||
|
"image": { "enabled": true },
|
||||||
"sub_worker": { "enabled": true },
|
"sub_worker": { "enabled": true },
|
||||||
"worker": { "enabled": false },
|
"worker": { "enabled": false },
|
||||||
"objective": { "enabled": true },
|
"objective": { "enabled": true },
|
||||||
@@ -998,6 +999,7 @@ fn apply_role_profile(
|
|||||||
value["feature"]["task"] = serde_json::json!({ "enabled": task });
|
value["feature"]["task"] = serde_json::json!({ "enabled": task });
|
||||||
value["feature"]["memory"] = serde_json::json!({ "enabled": memory });
|
value["feature"]["memory"] = serde_json::json!({ "enabled": memory });
|
||||||
value["feature"]["web"] = serde_json::json!({ "enabled": web });
|
value["feature"]["web"] = serde_json::json!({ "enabled": web });
|
||||||
|
value["feature"]["image"] = serde_json::json!({ "enabled": true });
|
||||||
value["feature"]["sub_worker"] = serde_json::json!({ "enabled": sub_worker });
|
value["feature"]["sub_worker"] = serde_json::json!({ "enabled": sub_worker });
|
||||||
value["feature"]["flow"] = serde_json::json!({ "enabled": slug == "coder" });
|
value["feature"]["flow"] = serde_json::json!({ "enabled": slug == "coder" });
|
||||||
value["feature"]["worker"] =
|
value["feature"]["worker"] =
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ impl Tool for WriteExtractedTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -153,6 +153,7 @@ impl From<LoggedItem> for Item {
|
|||||||
summary,
|
summary,
|
||||||
content,
|
content,
|
||||||
is_error,
|
is_error,
|
||||||
|
attachments: Vec::new(),
|
||||||
},
|
},
|
||||||
LoggedItem::Reasoning {
|
LoggedItem::Reasoning {
|
||||||
text,
|
text,
|
||||||
@@ -213,6 +214,9 @@ impl From<&ContentPart> for LoggedContentPart {
|
|||||||
fn from(part: &ContentPart) -> Self {
|
fn from(part: &ContentPart) -> Self {
|
||||||
match part {
|
match part {
|
||||||
ContentPart::Text { text } => Self::Text { text: text.clone() },
|
ContentPart::Text { text } => Self::Text { text: text.clone() },
|
||||||
|
ContentPart::Image { .. } => Self::Text {
|
||||||
|
text: part.as_text().to_string(),
|
||||||
|
},
|
||||||
ContentPart::Refusal { refusal } => Self::Refusal {
|
ContentPart::Refusal { refusal } => Self::Refusal {
|
||||||
refusal: refusal.clone(),
|
refusal: refusal.clone(),
|
||||||
},
|
},
|
||||||
@@ -370,6 +374,31 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tool_result_persistence_drops_binary_attachments() {
|
||||||
|
let original = Item::tool_result_item_with_attachments(
|
||||||
|
"call_image",
|
||||||
|
"attached",
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
vec![llm_engine::tool::Attachment::Image(
|
||||||
|
llm_engine::tool::ImageAttachment::new(
|
||||||
|
"image/png",
|
||||||
|
std::sync::Arc::<[u8]>::from(&b"secret-image-body"[..]),
|
||||||
|
),
|
||||||
|
)],
|
||||||
|
);
|
||||||
|
let logged: LoggedItem = (&original).into();
|
||||||
|
let json = serde_json::to_string(&logged).unwrap();
|
||||||
|
assert!(!json.contains("secret-image-body"));
|
||||||
|
assert!(!json.contains("attachments"));
|
||||||
|
|
||||||
|
match Item::from(logged) {
|
||||||
|
Item::ToolResult { attachments, .. } => assert!(attachments.is_empty()),
|
||||||
|
other => panic!("unexpected variant: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn message_serialization_uses_kind_tag() {
|
fn message_serialization_uses_kind_tag() {
|
||||||
let logged: LoggedItem = (&Item::assistant_message("hi")).into();
|
let logged: LoggedItem = (&Item::assistant_message("hi")).into();
|
||||||
|
|||||||
@@ -1703,6 +1703,7 @@ fn json_output(summary: String, value: impl Serialize) -> ToolOutput {
|
|||||||
ToolOutput {
|
ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string())),
|
content: Some(serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string())),
|
||||||
|
attachments: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -96,7 +96,11 @@ impl Tool for BashTool {
|
|||||||
} else {
|
} else {
|
||||||
Some(output.content)
|
Some(output.content)
|
||||||
};
|
};
|
||||||
Ok(ToolOutput { summary, content })
|
Ok(ToolOutput {
|
||||||
|
summary,
|
||||||
|
content,
|
||||||
|
attachments: Vec::new(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -85,6 +85,7 @@ impl Tool for EditTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(preview),
|
content: Some(preview),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ impl Tool for GlobTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: (!body.is_empty()).then_some(body),
|
content: (!body.is_empty()).then_some(body),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -120,6 +120,7 @@ impl Tool for GrepTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: (!result.output.is_empty()).then_some(result.output),
|
content: (!result.output.is_empty()).then_some(result.output),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ mod edit;
|
|||||||
mod glob;
|
mod glob;
|
||||||
mod grep;
|
mod grep;
|
||||||
mod read;
|
mod read;
|
||||||
|
mod view_image;
|
||||||
mod web;
|
mod web;
|
||||||
mod write;
|
mod write;
|
||||||
|
|
||||||
@@ -27,6 +28,7 @@ pub use glob::glob_tool;
|
|||||||
pub use grep::grep_tool;
|
pub use grep::grep_tool;
|
||||||
pub use read::read_tool;
|
pub use read::read_tool;
|
||||||
pub use tracker::Tracker;
|
pub use tracker::Tracker;
|
||||||
|
pub use view_image::view_image_tool;
|
||||||
pub use web::{web_fetch_tool, web_search_tool};
|
pub use web::{web_fetch_tool, web_search_tool};
|
||||||
pub use write::write_tool;
|
pub use write::write_tool;
|
||||||
|
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ impl Tool for ReadTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(rendered.body),
|
content: Some(rendered.body),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
//! `ViewImage` tool — attach a bounded image from the scoped Workdir.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use llm_engine::tool::{
|
||||||
|
Attachment, ImageAttachment, Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput,
|
||||||
|
};
|
||||||
|
use serde::Deserialize;
|
||||||
|
use workdir::{ReadRequest, WorkdirPath, WorkdirSessionHandle};
|
||||||
|
|
||||||
|
use crate::error::ToolsError;
|
||||||
|
|
||||||
|
/// Maximum image body accepted for one model request.
|
||||||
|
pub const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024;
|
||||||
|
|
||||||
|
const DESCRIPTION: &str = "Attach an image from the bound Workdir to the next model request. \
|
||||||
|
The path must be logical and Workdir-relative. Supported formats: PNG, JPEG, GIF, and WebP.";
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||||
|
struct ViewImageParams {
|
||||||
|
/// Logical path relative to the bound Workdir root.
|
||||||
|
path: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ViewImageTool {
|
||||||
|
session: WorkdirSessionHandle,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Tool for ViewImageTool {
|
||||||
|
async fn execute(
|
||||||
|
&self,
|
||||||
|
input_json: &str,
|
||||||
|
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||||
|
) -> Result<ToolOutput, ToolError> {
|
||||||
|
let input: ViewImageParams = serde_json::from_str(input_json).map_err(|error| {
|
||||||
|
ToolError::InvalidArgument(format!("invalid ViewImage input: {error}"))
|
||||||
|
})?;
|
||||||
|
let path = WorkdirPath::new(&input.path).map_err(ToolsError::from)?;
|
||||||
|
let result = self
|
||||||
|
.session
|
||||||
|
.read(ReadRequest {
|
||||||
|
path: path.clone(),
|
||||||
|
offset: 0,
|
||||||
|
limit: usize::MAX,
|
||||||
|
// The scoped provider enforces this cap while reading, rather
|
||||||
|
// than allocating an unbounded binary body first.
|
||||||
|
max_bytes: MAX_IMAGE_BYTES + 1,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(ToolsError::from)?;
|
||||||
|
|
||||||
|
if result.truncated || result.bytes.len() > MAX_IMAGE_BYTES {
|
||||||
|
return Err(ToolError::InvalidArgument(format!(
|
||||||
|
"image exceeds the {MAX_IMAGE_BYTES}-byte limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let mime_type = detect_image_mime(&result.bytes).ok_or_else(|| {
|
||||||
|
ToolError::InvalidArgument(
|
||||||
|
"unsupported image; expected PNG, JPEG, GIF, or WebP bytes".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let bytes = result.bytes.len();
|
||||||
|
|
||||||
|
Ok(ToolOutput {
|
||||||
|
summary: format!("Attached image {path} ({mime_type}, {bytes} bytes)"),
|
||||||
|
content: None,
|
||||||
|
attachments: vec![Attachment::Image(ImageAttachment::new(
|
||||||
|
mime_type,
|
||||||
|
Arc::<[u8]>::from(result.bytes),
|
||||||
|
))],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn view_image_tool(session: WorkdirSessionHandle) -> ToolDefinition {
|
||||||
|
Arc::new(move || {
|
||||||
|
let schema = schemars::schema_for!(ViewImageParams);
|
||||||
|
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||||
|
let meta = ToolMeta::new("ViewImage")
|
||||||
|
.description(DESCRIPTION)
|
||||||
|
.input_schema(schema_value);
|
||||||
|
let tool: Arc<dyn Tool> = Arc::new(ViewImageTool {
|
||||||
|
session: session.clone(),
|
||||||
|
});
|
||||||
|
(meta, tool)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn detect_image_mime(bytes: &[u8]) -> Option<&'static str> {
|
||||||
|
if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||||
|
Some("image/png")
|
||||||
|
} else if bytes.starts_with(&[0xff, 0xd8, 0xff]) {
|
||||||
|
Some("image/jpeg")
|
||||||
|
} else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") {
|
||||||
|
Some("image/gif")
|
||||||
|
} else if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WEBP" {
|
||||||
|
Some("image/webp")
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_supported_image_signatures_without_trusting_extensions() {
|
||||||
|
assert_eq!(
|
||||||
|
detect_image_mime(b"\x89PNG\r\n\x1a\nbody"),
|
||||||
|
Some("image/png")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
detect_image_mime(&[0xff, 0xd8, 0xff, 0xe0]),
|
||||||
|
Some("image/jpeg")
|
||||||
|
);
|
||||||
|
assert_eq!(detect_image_mime(b"GIF89abody"), Some("image/gif"));
|
||||||
|
assert_eq!(detect_image_mime(b"RIFF1234WEBPbody"), Some("image/webp"));
|
||||||
|
assert_eq!(detect_image_mime(b"not an image"), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1743,6 +1743,7 @@ fn json_output(value: Value) -> ToolOutput {
|
|||||||
ToolOutput {
|
ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -76,6 +76,7 @@ impl Tool for WriteTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use llm_engine::tool::{Tool, ToolDefinition, ToolMeta};
|
|||||||
use manifest::{Permission, Scope, ScopeConfig, ScopeRule};
|
use manifest::{Permission, Scope, ScopeConfig, ScopeRule};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use tools::{Tracker, core_builtin_tools};
|
use tools::{Tracker, core_builtin_tools, view_image_tool};
|
||||||
use workdir::{LocalWorkdirSession, WorkdirSessionHandle};
|
use workdir::{LocalWorkdirSession, WorkdirSessionHandle};
|
||||||
|
|
||||||
fn scope_with_spill(workspace: &Path, spill: &Path) -> Scope {
|
fn scope_with_spill(workspace: &Path, spill: &Path) -> Scope {
|
||||||
@@ -100,6 +100,31 @@ fn meta_has_description_and_schema() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn view_image_reads_scoped_bytes_without_serializing_them_as_text() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let spill = TempDir::new().unwrap();
|
||||||
|
let scope = scope_with_spill(dir.path(), spill.path());
|
||||||
|
let session: WorkdirSessionHandle =
|
||||||
|
Arc::new(LocalWorkdirSession::new(scope, dir.path().to_path_buf()));
|
||||||
|
let png = b"\x89PNG\r\n\x1a\nprivate-image-body";
|
||||||
|
std::fs::write(dir.path().join("image.png"), png).unwrap();
|
||||||
|
let definition = view_image_tool(session);
|
||||||
|
let (_meta, tool) = definition();
|
||||||
|
|
||||||
|
let output = call(&tool, json!({ "path": "image.png" })).await;
|
||||||
|
assert_eq!(output.attachments.len(), 1);
|
||||||
|
let llm_engine::tool::Attachment::Image(image) = &output.attachments[0];
|
||||||
|
assert_eq!(image.mime_type(), "image/png");
|
||||||
|
assert_eq!(image.data(), png);
|
||||||
|
let serialized = serde_json::to_string(&output).unwrap();
|
||||||
|
assert!(!serialized.contains("private-image-body"));
|
||||||
|
assert!(!serialized.contains("attachments"));
|
||||||
|
|
||||||
|
let escaped = call_err(&tool, json!({ "path": "../outside.png" })).await;
|
||||||
|
assert!(escaped.to_string().contains("scope") || escaped.to_string().contains("path"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn read_then_edit_then_read_roundtrip() {
|
async fn read_then_edit_then_read_roundtrip() {
|
||||||
let (dir, _spill, reg) = setup();
|
let (dir, _spill, reg) = setup();
|
||||||
|
|||||||
@@ -228,6 +228,8 @@ impl Tool for SearchSessionLogTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: (!content.is_empty()).then_some(content),
|
content: (!content.is_empty()).then_some(content),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -290,6 +292,8 @@ impl Tool for ReadSessionItemsTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: (!content.is_empty()).then_some(content),
|
content: (!content.is_empty()).then_some(content),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -395,6 +399,8 @@ impl Tool for MarkReadRequiredTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: None,
|
content: None,
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -423,6 +429,8 @@ impl Tool for AddReferenceTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("Added reference {}", params.file_path.display()),
|
summary: format!("Added reference {}", params.file_path.display()),
|
||||||
content: None,
|
content: None,
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -452,6 +460,8 @@ impl Tool for WriteSummaryTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: note.to_string(),
|
summary: note.to_string(),
|
||||||
content: None,
|
content: None,
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -648,6 +648,12 @@ where
|
|||||||
tracker.clone(),
|
tracker.clone(),
|
||||||
bash_output_dir,
|
bash_output_dir,
|
||||||
));
|
));
|
||||||
|
if feature_config.image.enabled && model_supports_image_attachments(&spawner_manifest.model)
|
||||||
|
{
|
||||||
|
worker
|
||||||
|
.engine_mut()
|
||||||
|
.register_tool(tools::view_image_tool(workdir.clone()));
|
||||||
|
}
|
||||||
(Some(workdir), Some(tracker))
|
(Some(workdir), Some(tracker))
|
||||||
} else {
|
} else {
|
||||||
(None, None)
|
(None, None)
|
||||||
@@ -1570,6 +1576,16 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn model_supports_image_attachments(model: &manifest::ModelManifest) -> bool {
|
||||||
|
manifest::model_catalog::resolve_model_manifest(model).is_ok_and(|model| {
|
||||||
|
model.capability.is_some_and(|capability| capability.vision)
|
||||||
|
&& matches!(
|
||||||
|
model.scheme,
|
||||||
|
manifest::SchemeKind::OpenaiChat | manifest::SchemeKind::OpenaiResponses
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn build_greeting<C, St>(worker: &Worker<C, St>) -> protocol::Greeting
|
fn build_greeting<C, St>(worker: &Worker<C, St>) -> protocol::Greeting
|
||||||
where
|
where
|
||||||
C: LlmClient,
|
C: LlmClient,
|
||||||
@@ -1649,6 +1665,20 @@ mod tests {
|
|||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use tokio::net::UnixListener;
|
use tokio::net::UnixListener;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn image_attachment_gate_requires_vision_and_supported_openai_scheme() {
|
||||||
|
let openai = manifest::ModelManifest {
|
||||||
|
ref_: Some("codex-oauth/gpt-5.6-sol".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let anthropic = manifest::ModelManifest {
|
||||||
|
ref_: Some("anthropic/claude-opus-4-8".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(model_supports_image_attachments(&openai));
|
||||||
|
assert!(!model_supports_image_attachments(&anthropic));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn pending_run_parent_origin_table() {
|
fn pending_run_parent_origin_table() {
|
||||||
assert!(PendingRun::Run(Vec::new()).is_parent_originated());
|
assert!(PendingRun::Run(Vec::new()).is_parent_originated());
|
||||||
|
|||||||
@@ -854,6 +854,7 @@ where
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(json_content(&items)?),
|
content: Some(json_content(&items)?),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -890,6 +891,8 @@ where
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(json_content(&result)?),
|
content: Some(json_content(&result)?),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -982,6 +985,7 @@ where
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("sent peer message to `{}`", input.name),
|
summary: format!("sent peer message to `{}`", input.name),
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -253,6 +253,7 @@ impl Tool for RequestFlowTransitionTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -422,6 +423,7 @@ impl Tool for FinishFlowVerificationTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: "Recorded complete Flow verification verdicts.".to_string(),
|
summary: "Recorded complete Flow verification verdicts.".to_string(),
|
||||||
content: Some("{\"accepted\":true}".to_string()),
|
content: Some("{\"accepted\":true}".to_string()),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -492,6 +492,7 @@ fn workdir_output<T: Serialize>(summary: String, value: &T) -> Result<ToolOutput
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(serde_json::to_string_pretty(value).map_err(decode_error)?),
|
content: Some(serde_json::to_string_pretty(value).map_err(decode_error)?),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -283,6 +283,7 @@ impl Tool for WorkspaceWorkerTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("{} completed", self.operation.tool_name()),
|
summary: format!("{} completed", self.operation.tool_name()),
|
||||||
content: Some(response.body),
|
content: Some(response.body),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -292,6 +292,7 @@ fn tool_output(output: MemoryToolOutput) -> ToolOutput {
|
|||||||
ToolOutput {
|
ToolOutput {
|
||||||
summary: output.summary,
|
summary: output.summary,
|
||||||
content: output.content,
|
content: output.content,
|
||||||
|
attachments: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -235,6 +235,7 @@ impl Tool for StageMemoryCandidateTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("Staged Memory candidate {staging_id}."),
|
summary: format!("Staged Memory candidate {staging_id}."),
|
||||||
content: Some(format!("staging_id: {staging_id}")),
|
content: Some(format!("staging_id: {staging_id}")),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -281,6 +282,7 @@ impl Tool for FinishMemoryExtractionTool {
|
|||||||
format!("Finished extraction with {actual} staged candidate(s).")
|
format!("Finished extraction with {actual} staged candidate(s).")
|
||||||
}),
|
}),
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,6 +41,8 @@ impl WorkspaceHttpObjectiveBackend {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("Listed {count} objective(s)"),
|
summary: format!("Listed {count} objective(s)"),
|
||||||
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
|
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -249,6 +251,8 @@ fn objective_output(summary: String, response: ObjectiveDetail) -> Result<ToolOu
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
|
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -384,6 +384,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -111,6 +111,8 @@ impl Tool for TaskListTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: list_overview(active_tasks.len(), tasks.len()),
|
summary: list_overview(active_tasks.len(), tasks.len()),
|
||||||
content: Some(render_task_list(&tasks)),
|
content: Some(render_task_list(&tasks)),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -131,6 +133,8 @@ impl Tool for TaskGetTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("Task {} ({}) {}", task.taskid, task.status, task.subject),
|
summary: format!("Task {} ({}) {}", task.taskid, task.status, task.subject),
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -170,6 +174,8 @@ fn task_output(summary: String, task: &TaskEntry) -> ToolOutput {
|
|||||||
ToolOutput {
|
ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(serde_json::to_string_pretty(task).unwrap_or_default()),
|
content: Some(serde_json::to_string_pretty(task).unwrap_or_default()),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -739,6 +739,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -781,6 +781,7 @@ fn render_list_resources_result(result: ListResourcesResult) -> Result<ToolOutpu
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -822,6 +823,7 @@ fn render_read_resource_result(result: ReadResourceResult) -> Result<ToolOutput,
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -876,6 +878,7 @@ fn render_list_prompts_result(result: ListPromptsResult) -> Result<ToolOutput, T
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -924,6 +927,7 @@ fn render_get_prompt_result(result: GetPromptResult) -> Result<ToolOutput, ToolE
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1242,6 +1246,7 @@ fn render_call_tool_result(
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary,
|
summary,
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4098,6 +4098,8 @@ impl PluginInstance {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("{tool_name}: {tool_calls}"),
|
summary: format!("{tool_name}: {tool_calls}"),
|
||||||
content: Some(String::from_utf8_lossy(&input).to_string()),
|
content: Some(String::from_utf8_lossy(&input).to_string()),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
PluginInstanceRuntime::ComponentInstance(runtime) => {
|
PluginInstanceRuntime::ComponentInstance(runtime) => {
|
||||||
@@ -5447,7 +5449,11 @@ fn decode_plugin_wasm_output(bytes: &[u8]) -> Result<ToolOutput, PluginWasmError
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
Ok(ToolOutput { summary, content })
|
Ok(ToolOutput {
|
||||||
|
summary,
|
||||||
|
content,
|
||||||
|
attachments: Vec::new(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn bounded_message(message: impl Into<String>) -> String {
|
fn bounded_message(message: impl Into<String>) -> String {
|
||||||
|
|||||||
@@ -328,6 +328,8 @@ impl Interceptor for WorkerInterceptor {
|
|||||||
output: ToolOutput {
|
output: ToolOutput {
|
||||||
summary: info.result.summary.clone(),
|
summary: info.result.summary.clone(),
|
||||||
content: info.result.content.clone(),
|
content: info.result.content.clone(),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
for hook in &self.registry.post_tool_call {
|
for hook in &self.registry.post_tool_call {
|
||||||
@@ -911,6 +913,8 @@ mod tests {
|
|||||||
ToolOutput {
|
ToolOutput {
|
||||||
summary: "ok".into(),
|
summary: "ok".into(),
|
||||||
content: Some("full".into()),
|
content: Some("full".into()),
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
meta: info.meta,
|
meta: info.meta,
|
||||||
|
|||||||
@@ -89,6 +89,8 @@ mod tests {
|
|||||||
output: ToolOutput {
|
output: ToolOutput {
|
||||||
summary: "result".to_string(),
|
summary: "result".to_string(),
|
||||||
content: None,
|
content: None,
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ impl Tool for SubWorkerListTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("listed {count} child SubWorker(s)"),
|
summary: format!("listed {count} child SubWorker(s)"),
|
||||||
content: Some(content),
|
content: Some(content),
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -124,6 +125,7 @@ impl Tool for SubWorkerSendTool {
|
|||||||
return Ok(ToolOutput {
|
return Ok(ToolOutput {
|
||||||
summary: format!("sent message to `{}`", input.name),
|
summary: format!("sent message to `{}`", input.name),
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(unknown_worker_err(&input.name))
|
Err(unknown_worker_err(&input.name))
|
||||||
@@ -177,6 +179,7 @@ impl Tool for SubWorkerStopTool {
|
|||||||
input.name
|
input.name
|
||||||
),
|
),
|
||||||
content: None,
|
content: None,
|
||||||
|
attachments: Vec::new(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(unknown_worker_err(&input.name))
|
Err(unknown_worker_err(&input.name))
|
||||||
|
|||||||
@@ -492,6 +492,8 @@ impl Tool for SubWorkerSpawnTool {
|
|||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
summary: format!("spawned internal worker `{}`", input.name),
|
summary: format!("spawned internal worker `{}`", input.name),
|
||||||
content: None,
|
content: None,
|
||||||
|
|
||||||
|
attachments: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ feature = {
|
|||||||
task = { enabled = true; };
|
task = { enabled = true; };
|
||||||
memory = { enabled = true; };
|
memory = { enabled = true; };
|
||||||
web = { enabled = true; };
|
web = { enabled = true; };
|
||||||
|
image = { enabled = true; };
|
||||||
sub_worker = { enabled = true; };
|
sub_worker = { enabled = true; };
|
||||||
worker = { enabled = false; };
|
worker = { enabled = false; };
|
||||||
objective = { enabled = true; };
|
objective = { enabled = true; };
|
||||||
|
|||||||
Reference in New Issue
Block a user