Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

feat: add scoped multimodal image attachments

Browse Source
This commit is contained in:
Hare
2026-08-10 21:49:39 +09:00
parent 64ced7dbad
commit 38b8f26a50
42 changed files with 718 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/llm-engine/src/llm_client/scheme/anthropic/request.rs
+3
View File
@@ -268,6 +268,9 @@ impl AnthropicScheme {
.iter()
.map(|p| match p {
ContentPart::Text { text } => AnthropicContentPart::text(text.clone()),
ContentPart::Image { .. } => {
AnthropicContentPart::text(p.as_text().to_string())
}
ContentPart::Refusal { refusal } => {
AnthropicContentPart::text(refusal.clone())
}
crates/llm-engine/src/llm_client/scheme/openai_chat/request.rs
+138 -10
View File
@@ -8,7 +8,7 @@ use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{Item, Role, ToolDefinition, parse_tool_arguments},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
};
use super::OpenAIScheme;
@@ -134,7 +134,7 @@ impl OpenAIScheme {
}
// Convert items to messages
messages.extend(self.convert_items_to_messages(&request.items));
messages.extend(self.convert_items_to_messages(&request.items, capability.vision));
let tools = request.tools.iter().map(|t| self.convert_tool(t)).collect();
@@ -185,7 +185,11 @@ impl OpenAIScheme {
/// - Assistant messages have role "assistant"
/// - Tool calls are within assistant messages as tool_calls array
/// - Tool results have role "tool" with tool_call_id
fn convert_items_to_messages(&self, items: &[Item]) -> Vec<OpenAIMessage> {
fn convert_items_to_messages(
&self,
items: &[Item],
supports_images: bool,
) -> Vec<OpenAIMessage> {
let mut messages = Vec::new();
let mut pending_tool_calls: Vec<OpenAIToolCall> = Vec::new();
let mut pending_assistant_text: Option<String> = None;
@@ -205,16 +209,45 @@ impl OpenAIScheme {
Role::Assistant => "assistant",
Role::System => "system",
};
let text_content: String = content
.iter()
.map(|p| p.as_text())
.collect::<Vec<_>>()
.join("");
let has_image = matches!(role, Role::User)
&& supports_images
&& content
.iter()
.any(|part| matches!(part, ContentPart::Image { .. }));
let message_content = if has_image {
OpenAIContent::Parts(
content
.iter()
.map(|part| match part {
ContentPart::Text { text } => {
OpenAIContentPart::Text { text: text.clone() }
}
ContentPart::Image { media_type, source } => {
OpenAIContentPart::ImageUrl {
image_url: ImageUrl {
url: image_data_url(media_type, source.data()),
},
}
}
ContentPart::Refusal { refusal } => OpenAIContentPart::Text {
text: refusal.clone(),
},
})
.collect(),
)
} else {
OpenAIContent::Text(
content
.iter()
.map(ContentPart::as_text)
.collect::<Vec<_>>()
.join(""),
)
};
messages.push(OpenAIMessage {
role: openai_role.to_string(),
content: Some(OpenAIContent::Text(text_content)),
content: Some(message_content),
tool_calls: vec![],
tool_call_id: None,
name: None,
@@ -334,6 +367,13 @@ mod tests {
}
}
fn vision_cap() -> ModelCapability {
ModelCapability {
vision: true,
..cap()
}
}
#[test]
fn test_build_simple_request() {
let scheme = OpenAIScheme::new();
@@ -439,4 +479,92 @@ mod tests {
assert_eq!(body.messages[1].tool_calls.len(), 1);
assert_eq!(body.messages[2].role, "tool");
}
#[test]
fn parallel_tool_results_precede_synthetic_image_message() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let request = Request::new()
.item(Item::tool_call("call_image", "ViewImage", "{}"))
.item(Item::tool_call("call_text", "Read", "{}"))
.item(Item::tool_result_item(
"call_image",
"Attached image",
None,
false,
))
.item(Item::tool_result_item(
"call_text",
"Read text",
None,
false,
))
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let json = serde_json::to_value(
&scheme
.build_request("gpt-4o", &request, &vision_cap())
.messages,
)
.unwrap();
assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool");
assert_eq!(json[2]["role"], "tool");
assert_eq!(json[3]["role"], "user");
assert_eq!(json[3]["content"][0]["type"], "image_url");
}
#[test]
fn tool_image_is_structured_as_following_user_content_without_persisting_bytes() {
let scheme = OpenAIScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let attachment = crate::tool::Attachment::Image(crate::tool::ImageAttachment::new(
"image/png",
image.clone(),
));
let item = Item::tool_result_item_with_attachments(
"call_image",
"Attached image",
None,
false,
vec![attachment],
);
let persisted = serde_json::to_string(&item).unwrap();
assert!(!persisted.contains("base64"));
assert!(!persisted.contains("attachments"));
let request = Request::new()
.item(Item::tool_call(
"call_image",
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-4o", &request, &vision_cap());
let json = serde_json::to_value(&body.messages).unwrap();
assert_eq!(json[0]["role"], "assistant");
assert_eq!(json[1]["role"], "tool");
assert_eq!(json[2]["role"], "user");
assert_eq!(json[2]["content"][0]["type"], "image_url");
assert!(
json[2]["content"][0]["image_url"]["url"]
.as_str()
.unwrap()
.starts_with("data:image/png;base64,")
);
let mut no_vision = cap();
no_vision.vision = false;
let disabled =
serde_json::to_string(&scheme.build_request("gpt-4o", &request, &no_vision)).unwrap();
assert!(!disabled.contains("data:image"));
}
}
crates/llm-engine/src/llm_client/scheme/openai_responses/request.rs
+65 -12
View File
@@ -10,7 +10,7 @@ use serde_json::Value;
use crate::llm_client::{
Request,
capability::{ModelCapability, ReasoningControl, ReasoningSupport},
types::{ContentPart, Item, Role, ToolDefinition, parse_tool_arguments},
types::{ContentPart, Item, Role, ToolDefinition, image_data_url, parse_tool_arguments},
};
use super::OpenAIResponsesScheme;
@@ -89,12 +89,7 @@ pub(crate) enum InputItem {
arguments: String,
},
/// function tool の結果（user 側）。
FunctionCallOutput {
call_id: String,
/// Responses は文字列 or 構造化 output を許すが、ここでは
/// `summary` + `content` を改行連結した文字列で送る。
output: String,
},
FunctionCallOutput { call_id: String, output: String },
/// reasoning item。`encrypted_content` があれば必ず添える。
Reasoning {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -118,6 +113,11 @@ pub(crate) enum InputItem {
pub(crate) enum InputContent {
/// user / developer 側のテキスト
InputText { text: String },
/// user 側の画像
InputImage {
image_url: String,
detail: &'static str,
},
/// assistant 側のテキスト
OutputText { text: String },
}
@@ -173,7 +173,7 @@ impl OpenAIResponsesScheme {
request: &Request,
capability: &ModelCapability,
) -> ResponsesRequest {
let input = convert_items_to_input(&request.items);
let input = convert_items_to_input(&request.items, capability.vision);
let tools = request.tools.iter().map(convert_tool).collect();
// Reasoning 投影: capability が Effort / Both をサポートし、かつ
@@ -234,7 +234,7 @@ impl OpenAIResponsesScheme {
}
/// `Item` 列を `input[]` に変換する。
fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
fn convert_items_to_input(items: &[Item], supports_images: bool) -> Vec<InputItem> {
let mut out = Vec::with_capacity(items.len());
for item in items {
match item {
@@ -247,8 +247,17 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
};
let parts: Vec<InputContent> = content
.iter()
.map(|p| match p {
.map(|part| match part {
ContentPart::Text { text } => text_variant(text.clone()),
ContentPart::Image { media_type, source }
if matches!(role, Role::User) && supports_images =>
{
InputContent::InputImage {
image_url: image_data_url(media_type, source.data()),
detail: "auto",
}
}
ContentPart::Image { .. } => text_variant(part.as_text().to_string()),
ContentPart::Refusal { refusal } => text_variant(refusal.clone()),
})
.collect();
@@ -277,13 +286,13 @@ fn convert_items_to_input(items: &[Item]) -> Vec<InputItem> {
content,
..
} => {
let text = match content {
let output = match content {
Some(c) => format!("{summary}\n{c}"),
None => summary.clone(),
};
out.push(InputItem::FunctionCallOutput {
call_id: call_id.clone(),
output: text,
output,
});
}
Item::Reasoning {
@@ -690,4 +699,48 @@ mod tests {
assert_eq!(json["tools"][0]["type"], "function");
assert_eq!(json["tools"][0]["name"], "t");
}
#[test]
fn synthetic_user_image_uses_responses_input_image_content() {
let scheme = OpenAIResponsesScheme::new();
let image = std::sync::Arc::<[u8]>::from(&b"\x89PNG\r\n\x1a\nbody"[..]);
let item = Item::tool_result_item_with_attachments(
"call_image",
"Attached image",
None,
false,
vec![crate::tool::Attachment::Image(
crate::tool::ImageAttachment::new("image/png", image.clone()),
)],
);
let req = Request::new()
.item(Item::tool_call(
"call_image",
"ViewImage",
r#"{"path":"a.png"}"#,
))
.item(item)
.item(Item::user_message_parts(vec![ContentPart::image(
"image/png",
image,
)]));
let body = scheme.build_request("gpt-5", &req, &cap_with_reasoning());
let json = serde_json::to_value(&body).unwrap();
assert_eq!(json["input"][1]["type"], "function_call_output");
assert_eq!(json["input"][2]["type"], "message");
assert_eq!(json["input"][2]["content"][0]["type"], "input_image");
assert!(
json["input"][2]["content"][0]["image_url"]
.as_str()
.unwrap()
.starts_with("data:image/png;base64,")
);
let mut no_vision = cap_with_reasoning();
no_vision.vision = false;
let disabled =
serde_json::to_string(&scheme.build_request("gpt-5", &req, &no_vision)).unwrap();
assert!(!disabled.contains("data:image"));
}
}
crates/llm-engine/src/llm_client/types.rs
+78 -1
View File
@@ -9,12 +9,19 @@
use std::{fmt, sync::Arc};
use crate::tool::Attachment;
use base64::Engine as _;
use serde::{Deserialize, Serialize};
fn is_false(value: &bool) -> bool {
!*value
}
pub(crate) fn image_data_url(media_type: &str, data: &[u8]) -> String {
let encoded = base64::engine::general_purpose::STANDARD.encode(data);
format!("data:{media_type};base64,{encoded}")
}
// ============================================================================
// Item - The core unit of conversation
// ============================================================================
@@ -117,6 +124,9 @@ pub enum Item {
/// Whether the tool result represents an execution error.
#[serde(default, skip_serializing_if = "is_false")]
is_error: bool,
/// Request-local structured payloads. Never serialized or persisted.
#[serde(skip, default)]
attachments: Vec<Attachment>,
},
/// Reasoning/thinking item
@@ -250,6 +260,17 @@ impl Item {
summary: impl Into<String>,
content: Option<String>,
is_error: bool,
) -> Self {
Self::tool_result_item_with_attachments(call_id, summary, content, is_error, Vec::new())
}
/// Create a tool result item with request-local structured attachments.
pub fn tool_result_item_with_attachments(
call_id: impl Into<String>,
summary: impl Into<String>,
content: Option<String>,
is_error: bool,
attachments: Vec<Attachment>,
) -> Self {
Self::ToolResult {
id: None,
@@ -257,6 +278,14 @@ impl Item {
summary: summary.into(),
content,
is_error,
attachments,
}
}
/// Drop request-local attachments after constructing the provider request.
pub fn clear_transient_attachments(&mut self) {
if let Self::ToolResult { attachments, .. } = self {
attachments.clear();
}
}
@@ -428,6 +457,38 @@ pub fn parse_tool_arguments(arguments: &str) -> serde_json::Value {
// Content Parts - Components within message items
// ============================================================================
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ImageSource {
bytes: usize,
#[serde(skip, default)]
data: Arc<[u8]>,
}
impl ImageSource {
pub fn new(data: Arc<[u8]>) -> Self {
Self {
bytes: data.len(),
data,
}
}
pub fn data(&self) -> &[u8] {
&self.data
}
pub fn bytes(&self) -> usize {
self.bytes
}
}
impl fmt::Debug for ImageSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ImageSource")
.field("bytes", &self.bytes)
.finish()
}
}
/// Content part within a message item
///
/// Text content is role-agnostic; the containing Item's Role determines
@@ -441,6 +502,12 @@ pub enum ContentPart {
text: String,
},
/// Request-local image content. The source bytes are never serialized.
Image {
media_type: String,
source: ImageSource,
},
/// Refusal content (for assistant messages)
Refusal {
/// The refusal message
@@ -461,10 +528,20 @@ impl ContentPart {
}
}
/// Get the text content regardless of type
pub fn image(media_type: impl Into<String>, data: Arc<[u8]>) -> Self {
Self::Image {
media_type: media_type.into(),
source: ImageSource::new(data),
}
}
/// Get a bounded textual projection. Image content is represented by an
/// explicit placeholder rather than silently becoming an empty string,
/// filesystem path, data URL, or base64 body.
pub fn as_text(&self) -> &str {
match self {
Self::Text { text } => text,
Self::Image { .. } => "[image attachment omitted]",
Self::Refusal { refusal } => refusal,
}
}