Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: add scoped multimodal image attachments

Browse Source
This commit is contained in:
Hare
2026-08-10 21:49:39 +09:00
parent 64ced7dbad
commit 38b8f26a50
42 changed files with 718 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/llm-engine/Cargo.toml
+2 -2
View File
@@ -7,7 +7,7 @@ license.workspace = true
[features]
default = []
codex = ["dep:base64", "dep:chrono"]
codex = ["dep:chrono"]
[dependencies]
serde = { workspace = true, features = ["derive"] }
@@ -21,7 +21,7 @@ tokio-util = "0.7"
reqwest = { version = "0.13", default-features = false, features = ["stream", "json", "native-tls", "http2"] }
eventsource-stream = "0.2"
zstd = "0.13"
base64 = { version = "0.22.1", optional = true }
base64 = "0.22.1"
chrono = { version = "0.4", default-features = false, features = ["serde", "clock"], optional = true }
llm-engine-macros = { workspace = true }
crates/llm-engine/src/engine.rs
+87 -5
View File
@@ -19,15 +19,19 @@ use crate::{
},
llm_client::{
ClientError, ConfigWarning, LlmClient, Request, RequestConfig, ResponseStream,
ToolDefinition, error::is_retryable, event::Event, retry::RetryPolicy,
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT, types::parse_tool_arguments,
ToolDefinition,
error::is_retryable,
event::Event,
retry::RetryPolicy,
transport::DEFAULT_FIRST_STREAM_EVENT_TIMEOUT,
types::{ContentPart, parse_tool_arguments},
},
state::{EngineState, Locked, Mutable},
timeline::event::{ErrorEvent, StatusEvent, UsageEvent},
timeline::{TextBlockCollector, ThinkingBlockCollector, Timeline, ToolCallCollector},
tool::{
ToolCall, ToolDefinition as EngineToolDefinition, ToolError, ToolExecutionContext,
ToolOutputLimits, ToolResult, truncate_content,
Attachment, ToolCall, ToolDefinition as EngineToolDefinition, ToolError,
ToolExecutionContext, ToolOutputLimits, ToolResult, truncate_content,
},
tool_server::{ToolServer, ToolServerHandle},
};
@@ -155,6 +159,33 @@ enum StreamCompletion {
Interrupted { reason: String },
}
fn project_transient_attachments(items: &mut Vec<Item>) {
let mut projected = Vec::with_capacity(items.len());
let mut pending_parts = Vec::new();
for mut item in items.drain(..) {
let is_tool_result = matches!(item, Item::ToolResult { .. });
if !is_tool_result && !pending_parts.is_empty() {
projected.push(Item::user_message_parts(std::mem::take(&mut pending_parts)));
}
if let Item::ToolResult { attachments, .. } = &item {
for attachment in attachments {
let Attachment::Image(image) = attachment;
pending_parts.push(ContentPart::image(
image.mime_type(),
Arc::from(image.data()),
));
}
}
item.clear_transient_attachments();
projected.push(item);
}
if !pending_parts.is_empty() {
projected.push(Item::user_message_parts(pending_parts));
}
*items = projected;
}
pub struct Engine<C: LlmClient, S: EngineState = Mutable> {
/// LLM client
client: C,
@@ -1249,6 +1280,8 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
cb(current_llm_call);
}
project_transient_attachments(&mut request_context);
// Stream LLM response
self.emit_lifecycle_trace(
current_turn,
@@ -1257,6 +1290,12 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
items_trace_payload(&request_context, tool_definitions.len(), None, false),
);
let request = self.build_request(&tool_definitions, &request_context);
// Structured attachments are single-use provider payloads. `request`
// owns its Arc-backed copy; history is cleared before any later turn
// can persist or resend the binary body.
for item in &mut self.history {
item.clear_transient_attachments();
}
self.emit_lifecycle_trace(
current_turn,
current_llm_call,
@@ -1618,11 +1657,12 @@ impl<C: LlmClient, S: EngineState> Engine<C, S> {
// Route per-result pushes through the callback path so
// observers see each tool result as it lands.
let items = results.into_iter().map(|result| {
Item::tool_result_item(
Item::tool_result_item_with_attachments(
&result.tool_use_id,
&result.summary,
result.content,
result.is_error,
result.attachments,
)
});
self.append_history_items(items)?;
@@ -2159,8 +2199,50 @@ fn item_kind(item: &Item) -> &'static str {
#[cfg(test)]
mod tests {
use super::*;
use crate::llm_client::types::Role;
use std::time::Duration;
#[test]
fn transient_tool_attachment_becomes_non_persistent_user_image_part() {
let body: Arc<[u8]> = Arc::from(&b"secret-image-body"[..]);
let mut items = vec![Item::tool_result_item_with_attachments(
"call_image",
"attached",
None,
false,
vec![Attachment::Image(crate::tool::ImageAttachment::new(
"image/png",
body.clone(),
))],
)];
project_transient_attachments(&mut items);
assert!(matches!(
&items[0],
Item::ToolResult { attachments, .. } if attachments.is_empty()
));
match &items[1] {
Item::Message {
role: Role::User,
content,
..
} => match &content[0] {
ContentPart::Image { media_type, source } => {
assert_eq!(media_type, "image/png");
assert_eq!(source.data(), body.as_ref());
assert_eq!(source.bytes(), body.len());
assert_eq!(content[0].as_text(), "[image attachment omitted]");
}
other => panic!("unexpected content part: {other:?}"),
},
other => panic!("unexpected projected item: {other:?}"),
}
let persisted = serde_json::to_string(&items).unwrap();
assert!(!persisted.contains("secret-image-body"));
assert!(!persisted.contains("base64"));
}
#[tokio::test]
async fn first_stream_event_timeout_returns_retryable_timeout() {
let stream: ResponseStream = Box::pin(futures::stream::pending());
crates/llm-engine/src/llm_client/scheme/anthropic/request.rs
+3
View File
@@ -268,6 +268,9 @@ impl AnthropicScheme {
.iter()
.map(|p| match p {
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
ContentPart::Image { .. } => {
AnthropicContentPart::text(p.as_text().to_string())
}
ContentPart::Refusal { refusal } => {
AnthropicContentPart::text(refusal.clone())
}
crates/llm-engine/src/llm_client/scheme/openai_chat/request.rs
+136 -8
View File
@@ -8,7 +8,7 @@ use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{Item, Role, ToolDefinition, parse_tool_arguments},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
};
use super::OpenAIScheme;
@@ -134,7 +134,7 @@ impl OpenAIScheme {
}
// Convert items to messages
messages.extend(self.convert_items_to_messages(&request.items));
messages.extend(self.convert_items_to_messages(&request.items, capability.vision));
let tools = request.tools.iter().map(|t| self.convert_tool(t)).collect();
@@ -185,7 +185,11 @@ impl OpenAIScheme {
/// - Assistant messages have role "assistant"
/// - Tool calls are within assistant messages as tool_calls array
/// - Tool results have role "tool" with tool_call_id
fn convert_items_to_messages(&self, items: &[Item]) -> Vec<OpenAIMessage> {
fn convert_items_to_messages(
&self,
items: &[Item],
supports_images: bool,
) -> Vec<OpenAIMessage> {
let mut messages = Vec::new();
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
let mut pending_assistant_text: Option<String> = None;
@@ -205,16 +209,45 @@ impl OpenAIScheme {
Role::Assistant => "assistant",
Role::System => "system",
};
let text_content: String = content
let has_image = matches!(role, Role::User)
&& supports_images
&& content
.iter()
.map(|p| p.as_text())
.any(|part| matches!(part, ContentPart::Image { .. }));
let message_content = if has_image {
OpenAIContent::Parts(
content
.iter()
.map(|part| match part {
ContentPart::Text { text } => {
OpenAIContentPart::Text { text: text.clone() }
}
ContentPart::Image { media_type, source } => {
OpenAIContentPart::ImageUrl {
image_url: ImageUrl {
url: image_data_url(media_type, source.data()),
},
}
}
ContentPart::Refusal { refusal } => OpenAIContentPart::Text {
text: refusal.clone(),
},
})
.collect(),
)
} else {
OpenAIContent::Text(
content
.iter()
.map(ContentPart::as_text)
.collect::<Vec<_>>()
.join("");
.join(""),
)
};
messages.push(OpenAIMessage {
role: openai_role.to_string(),
content: Some(OpenAIContent::Text(text_content)),
content: Some(message_content),
tool_calls: vec![],
tool_call_id: None,
name: None,
@@ -334,6 +367,13 @@ mod tests {
}
}
fn vision_cap() -> ModelCapability {
ModelCapability {
vision: true,
..cap()
}
}
#[test]
fn test_build_simple_request() {
let scheme = OpenAIScheme::new();
@@ -439,4 +479,92 @@ mod tests {
assert_eq!(body.messages[1].tool_calls.len(), 1);
assert_eq!(body.messages[2].role, "tool");
}
#[test]
fn parallel_tool_results_precede_synthetic_image_message() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let request = Request::new()
.item(Item::tool_call("call_image", "ViewImage", "{}"))
.item(Item::tool_call("call_text", "Read", "{}"))
.item(Item::tool_result_item(
"call_image",
"Attached image",
None,
false,
))
.item(Item::tool_result_item(
"call_text",
"Read text",
None,
false,
))
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let json = serde_json::to_value(
&scheme
.build_request("gpt-4o", &request, &vision_cap())
.messages,
)
.unwrap();
assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool");
assert_eq!(json[2]["role"], "tool");
assert_eq!(json[3]["role"], "user");
assert_eq!(json[3]["content"][0]["type"], "image_url");
}
#[test]
fn tool_image_is_structured_as_following_user_content_without_persisting_bytes() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
"image/png",
image.clone(),
));
let item = Item::tool_result_item_with_attachments(
"call_image",
"Attached image",
None,
false,
vec![attachment],
);
let persisted = serde_json::to_string(&item).unwrap();
assert!(!persisted.contains("base64"));
assert!(!persisted.contains("attachments"));
let request = Request::new()
.item(Item::tool_call(
"call_image",
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-4o", &request, &vision_cap());
let json = serde_json::to_value(&body.messages).unwrap();
assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool");
assert_eq!(json[2]["role"], "user");
assert_eq!(json[2]["content"][0]["type"], "image_url");
assert!(
json[2]["content"][0]["image_url"]["url"]
.as_str()
.unwrap()
.starts_with("data:image/png;base64,")
);
let mut no_vision = cap();
no_vision.vision = false;
let disabled =
serde_json::to_string(&scheme.build_request("gpt-4o", &request, &no_vision)).unwrap();
assert!(!disabled.contains("data:image"));
}
}
crates/llm-engine/src/llm_client/scheme/openai_responses/request.rs
+65 -12
View File
@@ -10,7 +10,7 @@ use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, parse_tool_arguments},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
};
use super::OpenAIResponsesScheme;
@@ -89,12 +89,7 @@ pub(crate) enum InputItem {
arguments: String,
},
/// function tool の結果（user 側）。
FunctionCallOutput {
call_id: String,
/// Responses は文字列 or 構造化 output を許すが、ここでは
/// `summary` + `content` を改行連結した文字列で送る。
output: String,
},
FunctionCallOutput { call_id: String, output: String },
/// reasoning item。`encrypted_content` があれば必ず添える。
Reasoning {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -118,6 +113,11 @@ pub(crate) enum InputItem {
pub(crate) enum InputContent {
/// user / developer 側のテキスト
InputText { text: String },
/// user 側の画像
InputImage {
image_url: String,
detail: &'static str,
},
/// assistant 側のテキスト
OutputText { text: String },
}
@@ -173,7 +173,7 @@ impl OpenAIResponsesScheme {
request: &Request,
capability: &ModelCapability,
) -> ResponsesRequest {
let input = convert_items_to_input(&request.items);
let input = convert_items_to_input(&request.items, capability.vision);
let tools = request.tools.iter().map(convert_tool).collect();
// Reasoning 投影: capability が Effort / Both をサポートし、かつ
@@ -234,7 +234,7 @@ impl OpenAIResponsesScheme {
}
/// `Item` 列を `input[]` に変換する。
fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputItem> {
let mut out = Vec::with_capacity(items.len());
for item in items {
match item {
@@ -247,8 +247,17 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
};
let parts: Vec<InputContent> = content
.iter()
.map(|p| match p {
.map(|part| match part {
ContentPart::Text { text } => text_variant(text.clone()),
ContentPart::Image { media_type, source }
if matches!(role, Role::User) && supports_images =>
{
InputContent::InputImage {
image_url: image_data_url(media_type, source.data()),
detail: "auto",
}
}
ContentPart::Image { .. } => text_variant(part.as_text().to_string()),
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
})
.collect();
@@ -277,13 +286,13 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
content,
..
} => {
let text = match content {
let output = match content {
Some(c) => format!("{summary}\n{c}"),
None => summary.clone(),
};
out.push(InputItem::FunctionCallOutput {
call_id: call_id.clone(),
output: text,
output,
});
}
Item::Reasoning {
@@ -690,4 +699,48 @@ mod tests {
assert_eq!(json["tools"][0]["type"], "function");
assert_eq!(json["tools"][0]["name"], "t");
}
#[test]
fn synthetic_user_image_uses_responses_input_image_content() {
let scheme = OpenAIResponsesScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let item = Item::tool_result_item_with_attachments(
"call_image",
"Attached image",
None,
false,
vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", image.clone()),
)],
);
let req = Request::new()
.item(Item::tool_call(
"call_image",
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
let json = serde_json::to_value(&body).unwrap();
assert_eq!(json["input"][1]["type"], "function_call_output");
assert_eq!(json["input"][2]["type"], "message");
assert_eq!(json["input"][2]["content"][0]["type"], "input_image");
assert!(
json["input"][2]["content"][0]["image_url"]
.as_str()
.unwrap()
.starts_with("data:image/png;base64,")
);
let mut no_vision = cap_with_reasoning();
no_vision.vision = false;
let disabled =
serde_json::to_string(&scheme.build_request("gpt-5", &req, &no_vision)).unwrap();
assert!(!disabled.contains("data:image"));
}
}
crates/llm-engine/src/llm_client/types.rs
+78 -1
View File
@@ -9,12 +9,19 @@
use std::{fmt, sync::Arc};
use crate::tool::Attachment;
use base64::Engine as _;
use serde::{Deserialize, Serialize};
fn is_false(value: &bool) -> bool {
!*value
}
pub(crate) fn image_data_url(media_type: &str, data: &[u8]) -> String {
let encoded = base64::engine::general_purpose::STANDARD.encode(data);
format!("data:{media_type};base64,{encoded}")
}
// ============================================================================
// Item - The core unit of conversation
// ============================================================================
@@ -117,6 +124,9 @@ pub enum Item {
/// Whether the tool result represents an execution error.
#[serde(default, skip_serializing_if = "is_false")]
is_error: bool,
/// Request-local structured payloads. Never serialized or persisted.
#[serde(skip, default)]
attachments: Vec<Attachment>,
},
/// Reasoning/thinking item
@@ -250,6 +260,17 @@ impl Item {
summary: impl Into<String>,
content: Option<String>,
is_error: bool,
) -> Self {
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
}
/// Create a tool result item with request-local structured attachments.
pub fn tool_result_item_with_attachments(
call_id: impl Into<String>,
summary: impl Into<String>,
content: Option<String>,
is_error: bool,
attachments: Vec<Attachment>,
) -> Self {
Self::ToolResult {
id: None,
@@ -257,6 +278,14 @@ impl Item {
summary: summary.into(),
content,
is_error,
attachments,
}
}
/// Drop request-local attachments after constructing the provider request.
pub fn clear_transient_attachments(&mut self) {
if let Self::ToolResult { attachments, .. } = self {
attachments.clear();
}
}
@@ -428,6 +457,38 @@ pub fn parse_tool_arguments(arguments: &str) -> serde_json::Value {
// Content Parts - Components within message items
// ============================================================================
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ImageSource {
bytes: usize,
#[serde(skip, default)]
data: Arc<[u8]>,
}
impl ImageSource {
pub fn new(data: Arc<[u8]>) -> Self {
Self {
bytes: data.len(),
data,
}
}
pub fn data(&self) -> &[u8] {
&self.data
}
pub fn bytes(&self) -> usize {
self.bytes
}
}
impl fmt::Debug for ImageSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ImageSource")
.field("bytes", &self.bytes)
.finish()
}
}
/// Content part within a message item
///
/// Text content is role-agnostic; the containing Item's Role determines
@@ -441,6 +502,12 @@ pub enum ContentPart {
text: String,
},
/// Request-local image content. The source bytes are never serialized.
Image {
media_type: String,
source: ImageSource,
},
/// Refusal content (for assistant messages)
Refusal {
/// The refusal message
@@ -461,10 +528,20 @@ impl ContentPart {
}
}
/// Get the text content regardless of type
pub fn image(media_type: impl Into<String>, data: Arc<[u8]>) -> Self {
Self::Image {
media_type: media_type.into(),
source: ImageSource::new(data),
}
}
/// Get a bounded textual projection. Image content is represented by an
/// explicit placeholder rather than silently becoming an empty string,
/// filesystem path, data URL, or base64 body.
pub fn as_text(&self) -> &str {
match self {
Self::Text { text } => text,
Self::Image { .. } => "[image attachment omitted]",
Self::Refusal { refusal } => refusal,
}
}
crates/llm-engine/src/tool.rs
+55 -5
View File
@@ -3,8 +3,7 @@
//! Traits for defining tools callable by LLM.
//! Usually auto-implemented using the `#[tool]` macro.
use std::collections::HashMap;
use std::sync::Arc;
use std::{collections::HashMap, fmt, sync::Arc};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
@@ -89,19 +88,63 @@ pub(crate) fn truncate_content(content: &mut String, limit: usize) {
content.push_str(&suffix_template.replace("%BYTES%", &dropped.to_string()));
}
#[derive(Clone, PartialEq, Eq)]
pub struct ImageAttachment {
mime_type: String,
data: Arc<[u8]>,
}
impl ImageAttachment {
pub fn new(mime_type: impl Into<String>, data: impl Into<Arc<[u8]>>) -> Self {
Self {
mime_type: mime_type.into(),
data: data.into(),
}
}
pub fn mime_type(&self) -> &str {
&self.mime_type
}
pub fn data(&self) -> &[u8] {
&self.data
}
}
impl fmt::Debug for ImageAttachment {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ImageAttachment")
.field("mime_type", &self.mime_type)
.field("bytes", &self.data.len())
.finish()
}
}
/// Request-local binary payload emitted by a tool.
///
/// Attachments are deliberately excluded from serde. They may be projected into
/// the immediately following provider request, but never into persisted history,
/// protocol events, logs, or telemetry.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Attachment {
Image(ImageAttachment),
}
/// Tool execution result.
///
/// Every output has a mandatory `summary` (1-2 lines) that persists in
/// conversation history even after pruning. The optional `content` carries
/// full details and is removed by the Prune mechanism when the context
/// grows too large.
/// full text details. `attachments` are request-local and are never serialized.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolOutput {
/// Short summary (1-2 lines). Always remains in history.
pub summary: String,
/// Detailed output. Removed by Prune when old enough.
/// Detailed text output. Removed by Prune when old enough.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
/// Structured binary payloads for the immediately following model request.
#[serde(skip, default)]
pub attachments: Vec<Attachment>,
}
impl From<String> for ToolOutput {
@@ -110,6 +153,7 @@ impl From<String> for ToolOutput {
ToolOutput {
summary: s,
content: None,
attachments: Vec::new(),
}
} else {
let lines = s.lines().count();
@@ -118,6 +162,7 @@ impl From<String> for ToolOutput {
ToolOutput {
summary,
content: Some(s),
attachments: Vec::new(),
}
}
}
@@ -364,6 +409,9 @@ pub struct ToolResult {
/// Whether this is an error
#[serde(default)]
pub is_error: bool,
/// Request-local structured payloads. Never serialized or persisted.
#[serde(skip, default)]
pub attachments: Vec<Attachment>,
}
impl ToolResult {
@@ -374,6 +422,7 @@ impl ToolResult {
summary: output.summary,
content: output.content,
is_error: false,
attachments: output.attachments,
}
}
@@ -384,6 +433,7 @@ impl ToolResult {
summary: message.into(),
content: None,
is_error: true,
attachments: Vec::new(),
}
}
}
crates/manifest/src/config.rs
+5
View File
@@ -83,6 +83,8 @@ pub struct FeatureConfigPartial {
#[serde(default)]
pub web: Option<FeatureFlagConfigPartial>,
#[serde(default)]
pub image: Option<FeatureFlagConfigPartial>,
#[serde(default)]
pub sub_worker: Option<FeatureFlagConfigPartial>,
#[serde(default)]
pub flow: Option<FeatureFlagConfigPartial>,
@@ -104,6 +106,7 @@ impl FeatureConfigPartial {
task: merge_option(self.task, other.task, FeatureFlagConfigPartial::merge),
memory: merge_option(self.memory, other.memory, MemoryFeatureConfigPartial::merge),
web: merge_option(self.web, other.web, FeatureFlagConfigPartial::merge),
image: merge_option(self.image, other.image, FeatureFlagConfigPartial::merge),
sub_worker: merge_option(
self.sub_worker,
other.sub_worker,
@@ -189,6 +192,7 @@ impl From<FeatureConfigPartial> for FeatureConfig {
.map(MemoryFeatureConfig::from)
.unwrap_or_default(),
web: value.web.map(FeatureFlagConfig::from).unwrap_or_default(),
image: value.image.map(FeatureFlagConfig::from).unwrap_or_default(),
sub_worker: value
.sub_worker
.map(FeatureFlagConfig::from)
@@ -282,6 +286,7 @@ impl From<FeatureConfig> for FeatureConfigPartial {
task: Some(value.task.into()),
memory: Some(value.memory.into()),
web: Some(value.web.into()),
image: Some(value.image.into()),
sub_worker: Some(value.sub_worker.into()),
flow: Some(value.flow.into()),
worker: Some(value.worker.into()),
crates/manifest/src/lib.rs
+3
View File
@@ -111,6 +111,8 @@ pub struct FeatureConfig {
#[serde(default)]
pub web: FeatureFlagConfig,
#[serde(default)]
pub image: FeatureFlagConfig,
#[serde(default)]
pub sub_worker: FeatureFlagConfig,
#[serde(default)]
pub flow: FeatureFlagConfig,
@@ -132,6 +134,7 @@ impl Default for FeatureConfig {
task: FeatureFlagConfig::disabled(),
memory: MemoryFeatureConfig::disabled(),
web: FeatureFlagConfig::disabled(),
image: FeatureFlagConfig::disabled(),
sub_worker: FeatureFlagConfig::disabled(),
flow: FeatureFlagConfig::disabled(),
worker: FeatureFlagConfig::disabled(),
crates/manifest/src/profile.rs
+2
View File
@@ -962,6 +962,7 @@ fn builtin_base_profile_artifact() -> serde_json::Value {
"task": { "enabled": true },
"memory": { "enabled": true },
"web": { "enabled": true },
"image": { "enabled": true },
"sub_worker": { "enabled": true },
"worker": { "enabled": false },
"objective": { "enabled": true },
@@ -998,6 +999,7 @@ fn apply_role_profile(
value["feature"]["task"] = serde_json::json!({ "enabled": task });
value["feature"]["memory"] = serde_json::json!({ "enabled": memory });
value["feature"]["web"] = serde_json::json!({ "enabled": web });
value["feature"]["image"] = serde_json::json!({ "enabled": true });
value["feature"]["sub_worker"] = serde_json::json!({ "enabled": sub_worker });
value["feature"]["flow"] = serde_json::json!({ "enabled": slug == "coder" });
value["feature"]["worker"] =
crates/memory/src/extract/tool.rs
+1
View File
@@ -85,6 +85,7 @@ impl Tool for WriteExtractedTool {
Ok(ToolOutput {
summary,
content: None,
attachments: Vec::new(),
})
}
}
crates/session-store/src/logged_item.rs
+29
View File
@@ -153,6 +153,7 @@ impl From<LoggedItem> for Item {
summary,
content,
is_error,
attachments: Vec::new(),
},
LoggedItem::Reasoning {
text,
@@ -213,6 +214,9 @@ impl From<&ContentPart> for LoggedContentPart {
fn from(part: &ContentPart) -> Self {
match part {
ContentPart::Text { text } => Self::Text { text: text.clone() },
ContentPart::Image { .. } => Self::Text {
text: part.as_text().to_string(),
},
ContentPart::Refusal { refusal } => Self::Refusal {
refusal: refusal.clone(),
},
@@ -370,6 +374,31 @@ mod tests {
}
}
#[test]
fn tool_result_persistence_drops_binary_attachments() {
let original = Item::tool_result_item_with_attachments(
"call_image",
"attached",
None,
false,
vec![llm_engine::tool::Attachment::Image(
llm_engine::tool::ImageAttachment::new(
"image/png",
std::sync::Arc::<[u8]>::from(&b"secret-image-body"[..]),
),
)],
);
let logged: LoggedItem = (&original).into();
let json = serde_json::to_string(&logged).unwrap();
assert!(!json.contains("secret-image-body"));
assert!(!json.contains("attachments"));
match Item::from(logged) {
Item::ToolResult { attachments, .. } => assert!(attachments.is_empty()),
other => panic!("unexpected variant: {other:?}"),
}
}
#[test]
fn message_serialization_uses_kind_tag() {
let logged: LoggedItem = (&Item::assistant_message("hi")).into();
crates/ticket/src/tool.rs
+1
View File
@@ -1703,6 +1703,7 @@ fn json_output(summary: String, value: impl Serialize) -> ToolOutput {
ToolOutput {
summary,
content: Some(serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string())),
attachments: Vec::new(),
}
}
crates/tools/src/bash.rs
+5 -1
View File
@@ -96,7 +96,11 @@ impl Tool for BashTool {
} else {
Some(output.content)
};
Ok(ToolOutput { summary, content })
Ok(ToolOutput {
summary,
content,
attachments: Vec::new(),
})
}
}
crates/tools/src/edit.rs
+1
View File
@@ -85,6 +85,7 @@ impl Tool for EditTool {
Ok(ToolOutput {
summary,
content: Some(preview),
attachments: Vec::new(),
})
}
}
crates/tools/src/glob.rs
+1
View File
@@ -69,6 +69,7 @@ impl Tool for GlobTool {
Ok(ToolOutput {
summary,
content: (!body.is_empty()).then_some(body),
attachments: Vec::new(),
})
}
}
crates/tools/src/grep.rs
+1
View File
@@ -120,6 +120,7 @@ impl Tool for GrepTool {
Ok(ToolOutput {
summary,
content: (!result.output.is_empty()).then_some(result.output),
attachments: Vec::new(),
})
}
}
crates/tools/src/lib.rs
+2
View File
@@ -17,6 +17,7 @@ mod edit;
mod glob;
mod grep;
mod read;
mod view_image;
mod web;
mod write;
@@ -27,6 +28,7 @@ pub use glob::glob_tool;
pub use grep::grep_tool;
pub use read::read_tool;
pub use tracker::Tracker;
pub use view_image::view_image_tool;
pub use web::{web_fetch_tool, web_search_tool};
pub use write::write_tool;
crates/tools/src/read.rs
+1
View File
@@ -86,6 +86,7 @@ impl Tool for ReadTool {
Ok(ToolOutput {
summary,
content: Some(rendered.body),
attachments: Vec::new(),
})
}
}
crates/tools/src/view_image.rs
+123
View File
@@ -0,0 +1,123 @@
//! `ViewImage` tool — attach a bounded image from the scoped Workdir.
use std::sync::Arc;
use async_trait::async_trait;
use llm_engine::tool::{
Attachment, ImageAttachment, Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput,
};
use serde::Deserialize;
use workdir::{ReadRequest, WorkdirPath, WorkdirSessionHandle};
use crate::error::ToolsError;
/// Maximum image body accepted for one model request.
pub const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024;
const DESCRIPTION: &str = "Attach an image from the bound Workdir to the next model request. \
The path must be logical and Workdir-relative. Supported formats: PNG, JPEG, GIF, and WebP.";
#[derive(Debug, Deserialize, schemars::JsonSchema)]
struct ViewImageParams {
/// Logical path relative to the bound Workdir root.
path: String,
}
struct ViewImageTool {
session: WorkdirSessionHandle,
}
#[async_trait]
impl Tool for ViewImageTool {
async fn execute(
&self,
input_json: &str,
_ctx: llm_engine::tool::ToolExecutionContext,
) -> Result<ToolOutput, ToolError> {
let input: ViewImageParams = serde_json::from_str(input_json).map_err(|error| {
ToolError::InvalidArgument(format!("invalid ViewImage input: {error}"))
})?;
let path = WorkdirPath::new(&input.path).map_err(ToolsError::from)?;
let result = self
.session
.read(ReadRequest {
path: path.clone(),
offset: 0,
limit: usize::MAX,
// The scoped provider enforces this cap while reading, rather
// than allocating an unbounded binary body first.
max_bytes: MAX_IMAGE_BYTES + 1,
})
.await
.map_err(ToolsError::from)?;
if result.truncated || result.bytes.len() > MAX_IMAGE_BYTES {
return Err(ToolError::InvalidArgument(format!(
"image exceeds the {MAX_IMAGE_BYTES}-byte limit"
)));
}
let mime_type = detect_image_mime(&result.bytes).ok_or_else(|| {
ToolError::InvalidArgument(
"unsupported image; expected PNG, JPEG, GIF, or WebP bytes".to_string(),
)
})?;
let bytes = result.bytes.len();
Ok(ToolOutput {
summary: format!("Attached image {path} ({mime_type}, {bytes} bytes)"),
content: None,
attachments: vec![Attachment::Image(ImageAttachment::new(
mime_type,
Arc::<[u8]>::from(result.bytes),
))],
})
}
}
pub fn view_image_tool(session: WorkdirSessionHandle) -> ToolDefinition {
Arc::new(move || {
let schema = schemars::schema_for!(ViewImageParams);
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
let meta = ToolMeta::new("ViewImage")
.description(DESCRIPTION)
.input_schema(schema_value);
let tool: Arc<dyn Tool> = Arc::new(ViewImageTool {
session: session.clone(),
});
(meta, tool)
})
}
fn detect_image_mime(bytes: &[u8]) -> Option<&'static str> {
if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
Some("image/png")
} else if bytes.starts_with(&[0xff, 0xd8, 0xff]) {
Some("image/jpeg")
} else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") {
Some("image/gif")
} else if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WEBP" {
Some("image/webp")
} else {
None
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_supported_image_signatures_without_trusting_extensions() {
assert_eq!(
detect_image_mime(b"\x89PNG\r\n\x1a\nbody"),
Some("image/png")
);
assert_eq!(
detect_image_mime(&[0xff, 0xd8, 0xff, 0xe0]),
Some("image/jpeg")
);
assert_eq!(detect_image_mime(b"GIF89abody"), Some("image/gif"));
assert_eq!(detect_image_mime(b"RIFF1234WEBPbody"), Some("image/webp"));
assert_eq!(detect_image_mime(b"not an image"), None);
}
}
crates/tools/src/web.rs
+1
View File
@@ -1743,6 +1743,7 @@ fn json_output(value: Value) -> ToolOutput {
ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
}
}
crates/tools/src/write.rs
+1
View File
@@ -76,6 +76,7 @@ impl Tool for WriteTool {
Ok(ToolOutput {
summary,
content: None,
attachments: Vec::new(),
})
}
}
crates/tools/tests/integration.rs
+26 -1
View File
@@ -11,7 +11,7 @@ use llm_engine::tool::{Tool, ToolDefinition, ToolMeta};
use manifest::{Permission, Scope, ScopeConfig, ScopeRule};
use serde_json::json;
use tempfile::TempDir;
use tools::{Tracker, core_builtin_tools};
use tools::{Tracker, core_builtin_tools, view_image_tool};
use workdir::{LocalWorkdirSession, WorkdirSessionHandle};
fn scope_with_spill(workspace: &Path, spill: &Path) -> Scope {
@@ -100,6 +100,31 @@ fn meta_has_description_and_schema() {
}
}
#[tokio::test]
async fn view_image_reads_scoped_bytes_without_serializing_them_as_text() {
let dir = TempDir::new().unwrap();
let spill = TempDir::new().unwrap();
let scope = scope_with_spill(dir.path(), spill.path());
let session: WorkdirSessionHandle =
Arc::new(LocalWorkdirSession::new(scope, dir.path().to_path_buf()));
let png = b"\x89PNG\r\n\x1a\nprivate-image-body";
std::fs::write(dir.path().join("image.png"), png).unwrap();
let definition = view_image_tool(session);
let (_meta, tool) = definition();
let output = call(&tool, json!({ "path": "image.png" })).await;
assert_eq!(output.attachments.len(), 1);
let llm_engine::tool::Attachment::Image(image) = &output.attachments[0];
assert_eq!(image.mime_type(), "image/png");
assert_eq!(image.data(), png);
let serialized = serde_json::to_string(&output).unwrap();
assert!(!serialized.contains("private-image-body"));
assert!(!serialized.contains("attachments"));
let escaped = call_err(&tool, json!({ "path": "../outside.png" })).await;
assert!(escaped.to_string().contains("scope") || escaped.to_string().contains("path"));
}
#[tokio::test]
async fn read_then_edit_then_read_roundtrip() {
let (dir, _spill, reg) = setup();
crates/worker/src/compact/worker.rs
+10
View File
@@ -228,6 +228,8 @@ impl Tool for SearchSessionLogTool {
Ok(ToolOutput {
summary,
content: (!content.is_empty()).then_some(content),
attachments: Vec::new(),
})
}
}
@@ -290,6 +292,8 @@ impl Tool for ReadSessionItemsTool {
Ok(ToolOutput {
summary,
content: (!content.is_empty()).then_some(content),
attachments: Vec::new(),
})
}
}
@@ -395,6 +399,8 @@ impl Tool for MarkReadRequiredTool {
Ok(ToolOutput {
summary,
content: None,
attachments: Vec::new(),
})
}
}
@@ -423,6 +429,8 @@ impl Tool for AddReferenceTool {
Ok(ToolOutput {
summary: format!("Added reference {}", params.file_path.display()),
content: None,
attachments: Vec::new(),
})
}
}
@@ -452,6 +460,8 @@ impl Tool for WriteSummaryTool {
Ok(ToolOutput {
summary: note.to_string(),
content: None,
attachments: Vec::new(),
})
}
}
crates/worker/src/controller.rs
+30
View File
@@ -648,6 +648,12 @@ where
tracker.clone(),
bash_output_dir,
));
if feature_config.image.enabled && model_supports_image_attachments(&spawner_manifest.model)
{
worker
.engine_mut()
.register_tool(tools::view_image_tool(workdir.clone()));
}
(Some(workdir), Some(tracker))
} else {
(None, None)
@@ -1570,6 +1576,16 @@ where
}
}
fn model_supports_image_attachments(model: &manifest::ModelManifest) -> bool {
manifest::model_catalog::resolve_model_manifest(model).is_ok_and(|model| {
model.capability.is_some_and(|capability| capability.vision)
&& matches!(
model.scheme,
manifest::SchemeKind::OpenaiChat | manifest::SchemeKind::OpenaiResponses
)
})
}
fn build_greeting<C, St>(worker: &Worker<C, St>) -> protocol::Greeting
where
C: LlmClient,
@@ -1649,6 +1665,20 @@ mod tests {
use tempfile::TempDir;
use tokio::net::UnixListener;
#[test]
fn image_attachment_gate_requires_vision_and_supported_openai_scheme() {
let openai = manifest::ModelManifest {
ref_: Some("codex-oauth/gpt-5.6-sol".to_string()),
..Default::default()
};
let anthropic = manifest::ModelManifest {
ref_: Some("anthropic/claude-opus-4-8".to_string()),
..Default::default()
};
assert!(model_supports_image_attachments(&openai));
assert!(!model_supports_image_attachments(&anthropic));
}
#[test]
fn pending_run_parent_origin_table() {
assert!(PendingRun::Run(Vec::new()).is_parent_originated());
crates/worker/src/discovery.rs
+4
View File
@@ -854,6 +854,7 @@ where
Ok(ToolOutput {
summary,
content: Some(json_content(&items)?),
attachments: Vec::new(),
})
}
}
@@ -890,6 +891,8 @@ where
Ok(ToolOutput {
summary,
content: Some(json_content(&result)?),
attachments: Vec::new(),
})
}
}
@@ -982,6 +985,7 @@ where
Ok(ToolOutput {
summary: format!("sent peer message to `{}`", input.name),
content: None,
attachments: Vec::new(),
})
}
}
crates/worker/src/feature/builtin/flow_transition.rs
+2
View File
@@ -253,6 +253,7 @@ impl Tool for RequestFlowTransitionTool {
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
}
@@ -422,6 +423,7 @@ impl Tool for FinishFlowVerificationTool {
Ok(ToolOutput {
summary: "Recorded complete Flow verification verdicts.".to_string(),
content: Some("{\"accepted\":true}".to_string()),
attachments: Vec::new(),
})
}
}
crates/worker/src/feature/builtin/manage_workdir.rs
+1
View File
@@ -492,6 +492,7 @@ fn workdir_output<T: Serialize>(summary: String, value: &T) -> Result<ToolOutput
Ok(ToolOutput {
summary,
content: Some(serde_json::to_string_pretty(value).map_err(decode_error)?),
attachments: Vec::new(),
})
}
crates/worker/src/feature/builtin/manage_worker.rs
+1
View File
@@ -283,6 +283,7 @@ impl Tool for WorkspaceWorkerTool {
Ok(ToolOutput {
summary: format!("{} completed", self.operation.tool_name()),
content: Some(response.body),
attachments: Vec::new(),
})
}
}
crates/worker/src/feature/builtin/memory.rs
+1
View File
@@ -292,6 +292,7 @@ fn tool_output(output: MemoryToolOutput) -> ToolOutput {
ToolOutput {
summary: output.summary,
content: output.content,
attachments: Vec::new(),
}
}
crates/worker/src/feature/builtin/memory_extract.rs
+2
View File
@@ -235,6 +235,7 @@ impl Tool for StageMemoryCandidateTool {
Ok(ToolOutput {
summary: format!("Staged Memory candidate {staging_id}."),
content: Some(format!("staging_id: {staging_id}")),
attachments: Vec::new(),
})
}
}
@@ -281,6 +282,7 @@ impl Tool for FinishMemoryExtractionTool {
format!("Finished extraction with {actual} staged candidate(s).")
}),
content: None,
attachments: Vec::new(),
})
}
}
crates/worker/src/feature/builtin/objective.rs
+4
View File
@@ -41,6 +41,8 @@ impl WorkspaceHttpObjectiveBackend {
Ok(ToolOutput {
summary: format!("Listed {count} objective(s)"),
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
attachments: Vec::new(),
})
}
@@ -249,6 +251,8 @@ fn objective_output(summary: String, response: ObjectiveDetail) -> Result<ToolOu
Ok(ToolOutput {
summary,
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
attachments: Vec::new(),
})
}
crates/worker/src/feature/builtin/session_explore.rs
+1
View File
@@ -384,6 +384,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
crates/worker/src/feature/builtin/task/tool_impl.rs
+6
View File
@@ -111,6 +111,8 @@ impl Tool for TaskListTool {
Ok(ToolOutput {
summary: list_overview(active_tasks.len(), tasks.len()),
content: Some(render_task_list(&tasks)),
attachments: Vec::new(),
})
}
}
@@ -131,6 +133,8 @@ impl Tool for TaskGetTool {
Ok(ToolOutput {
summary: format!("Task {} ({}) {}", task.taskid, task.status, task.subject),
content: Some(content),
attachments: Vec::new(),
})
}
}
@@ -170,6 +174,8 @@ fn task_output(summary: String, task: &TaskEntry) -> ToolOutput {
ToolOutput {
summary,
content: Some(serde_json::to_string_pretty(task).unwrap_or_default()),
attachments: Vec::new(),
}
}
crates/worker/src/feature/builtin/worker_observation.rs
+1
View File
@@ -739,6 +739,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
crates/worker/src/feature/mcp.rs
+5
View File
@@ -781,6 +781,7 @@ fn render_list_resources_result(result: ListResourcesResult) -> Result<ToolOutpu
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -822,6 +823,7 @@ fn render_read_resource_result(result: ReadResourceResult) -> Result<ToolOutput,
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -876,6 +878,7 @@ fn render_list_prompts_result(result: ListPromptsResult) -> Result<ToolOutput, T
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -924,6 +927,7 @@ fn render_get_prompt_result(result: GetPromptResult) -> Result<ToolOutput, ToolE
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -1242,6 +1246,7 @@ fn render_call_tool_result(
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
crates/worker/src/feature/plugin.rs
+7 -1
View File
@@ -4098,6 +4098,8 @@ impl PluginInstance {
Ok(ToolOutput {
summary: format!("{tool_name}: {tool_calls}"),
content: Some(String::from_utf8_lossy(&input).to_string()),
attachments: Vec::new(),
})
}
PluginInstanceRuntime::ComponentInstance(runtime) => {
@@ -5447,7 +5449,11 @@ fn decode_plugin_wasm_output(bytes: &[u8]) -> Result<ToolOutput, PluginWasmError
));
}
};
Ok(ToolOutput { summary, content })
Ok(ToolOutput {
summary,
content,
attachments: Vec::new(),
})
}
fn bounded_message(message: impl Into<String>) -> String {
crates/worker/src/ipc/interceptor.rs
+4
View File
@@ -328,6 +328,8 @@ impl Interceptor for WorkerInterceptor {
output: ToolOutput {
summary: info.result.summary.clone(),
content: info.result.content.clone(),
attachments: Vec::new(),
},
};
for hook in &self.registry.post_tool_call {
@@ -911,6 +913,8 @@ mod tests {
ToolOutput {
summary: "ok".into(),
content: Some("full".into()),
attachments: Vec::new(),
},
),
meta: info.meta,
crates/worker/src/shutdown_after_idle.rs
+2
View File
@@ -89,6 +89,8 @@ mod tests {
output: ToolOutput {
summary: "result".to_string(),
content: None,
attachments: Vec::new(),
},
}
}
crates/worker/src/spawn/comm_tools.rs
+3
View File
@@ -69,6 +69,7 @@ impl Tool for SubWorkerListTool {
Ok(ToolOutput {
summary: format!("listed {count} child SubWorker(s)"),
content: Some(content),
attachments: Vec::new(),
})
}
}
@@ -124,6 +125,7 @@ impl Tool for SubWorkerSendTool {
return Ok(ToolOutput {
summary: format!("sent message to `{}`", input.name),
content: None,
attachments: Vec::new(),
});
}
Err(unknown_worker_err(&input.name))
@@ -177,6 +179,7 @@ impl Tool for SubWorkerStopTool {
input.name
),
content: None,
attachments: Vec::new(),
});
}
Err(unknown_worker_err(&input.name))
crates/worker/src/spawn/tool.rs
+2
View File
@@ -492,6 +492,8 @@ impl Tool for SubWorkerSpawnTool {
Ok(ToolOutput {
summary: format!("spawned internal worker `{}`", input.name),
content: None,
attachments: Vec::new(),
})
}
}
resources/profiles/base.dcdl
+1
View File
@@ -25,6 +25,7 @@ feature = {
task = { enabled = true; };
memory = { enabled = true; };
web = { enabled = true; };
image = { enabled = true; };
sub_worker = { enabled = true; };
worker = { enabled = false; };
objective = { enabled = true; };