feat: add scoped multimodal image attachments

This commit is contained in:
2026-08-10 21:49:39 +09:00
parent 64ced7dbad
commit 38b8f26a50
42 changed files with 718 additions and 38 deletions
+10
View File
@@ -228,6 +228,8 @@ impl Tool for SearchSessionLogTool {
Ok(ToolOutput {
summary,
content: (!content.is_empty()).then_some(content),
attachments: Vec::new(),
})
}
}
@@ -290,6 +292,8 @@ impl Tool for ReadSessionItemsTool {
Ok(ToolOutput {
summary,
content: (!content.is_empty()).then_some(content),
attachments: Vec::new(),
})
}
}
@@ -395,6 +399,8 @@ impl Tool for MarkReadRequiredTool {
Ok(ToolOutput {
summary,
content: None,
attachments: Vec::new(),
})
}
}
@@ -423,6 +429,8 @@ impl Tool for AddReferenceTool {
Ok(ToolOutput {
summary: format!("Added reference {}", params.file_path.display()),
content: None,
attachments: Vec::new(),
})
}
}
@@ -452,6 +460,8 @@ impl Tool for WriteSummaryTool {
Ok(ToolOutput {
summary: note.to_string(),
content: None,
attachments: Vec::new(),
})
}
}
+30
View File
@@ -648,6 +648,12 @@ where
tracker.clone(),
bash_output_dir,
));
if feature_config.image.enabled && model_supports_image_attachments(&spawner_manifest.model)
{
worker
.engine_mut()
.register_tool(tools::view_image_tool(workdir.clone()));
}
(Some(workdir), Some(tracker))
} else {
(None, None)
@@ -1570,6 +1576,16 @@ where
}
}
fn model_supports_image_attachments(model: &manifest::ModelManifest) -> bool {
manifest::model_catalog::resolve_model_manifest(model).is_ok_and(|model| {
model.capability.is_some_and(|capability| capability.vision)
&& matches!(
model.scheme,
manifest::SchemeKind::OpenaiChat | manifest::SchemeKind::OpenaiResponses
)
})
}
fn build_greeting<C, St>(worker: &Worker<C, St>) -> protocol::Greeting
where
C: LlmClient,
@@ -1649,6 +1665,20 @@ mod tests {
use tempfile::TempDir;
use tokio::net::UnixListener;
#[test]
fn image_attachment_gate_requires_vision_and_supported_openai_scheme() {
let openai = manifest::ModelManifest {
ref_: Some("codex-oauth/gpt-5.6-sol".to_string()),
..Default::default()
};
let anthropic = manifest::ModelManifest {
ref_: Some("anthropic/claude-opus-4-8".to_string()),
..Default::default()
};
assert!(model_supports_image_attachments(&openai));
assert!(!model_supports_image_attachments(&anthropic));
}
#[test]
fn pending_run_parent_origin_table() {
assert!(PendingRun::Run(Vec::new()).is_parent_originated());
+4
View File
@@ -854,6 +854,7 @@ where
Ok(ToolOutput {
summary,
content: Some(json_content(&items)?),
attachments: Vec::new(),
})
}
}
@@ -890,6 +891,8 @@ where
Ok(ToolOutput {
summary,
content: Some(json_content(&result)?),
attachments: Vec::new(),
})
}
}
@@ -982,6 +985,7 @@ where
Ok(ToolOutput {
summary: format!("sent peer message to `{}`", input.name),
content: None,
attachments: Vec::new(),
})
}
}
@@ -253,6 +253,7 @@ impl Tool for RequestFlowTransitionTool {
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
}
@@ -422,6 +423,7 @@ impl Tool for FinishFlowVerificationTool {
Ok(ToolOutput {
summary: "Recorded complete Flow verification verdicts.".to_string(),
content: Some("{\"accepted\":true}".to_string()),
attachments: Vec::new(),
})
}
}
@@ -492,6 +492,7 @@ fn workdir_output<T: Serialize>(summary: String, value: &T) -> Result<ToolOutput
Ok(ToolOutput {
summary,
content: Some(serde_json::to_string_pretty(value).map_err(decode_error)?),
attachments: Vec::new(),
})
}
@@ -283,6 +283,7 @@ impl Tool for WorkspaceWorkerTool {
Ok(ToolOutput {
summary: format!("{} completed", self.operation.tool_name()),
content: Some(response.body),
attachments: Vec::new(),
})
}
}
@@ -292,6 +292,7 @@ fn tool_output(output: MemoryToolOutput) -> ToolOutput {
ToolOutput {
summary: output.summary,
content: output.content,
attachments: Vec::new(),
}
}
@@ -235,6 +235,7 @@ impl Tool for StageMemoryCandidateTool {
Ok(ToolOutput {
summary: format!("Staged Memory candidate {staging_id}."),
content: Some(format!("staging_id: {staging_id}")),
attachments: Vec::new(),
})
}
}
@@ -281,6 +282,7 @@ impl Tool for FinishMemoryExtractionTool {
format!("Finished extraction with {actual} staged candidate(s).")
}),
content: None,
attachments: Vec::new(),
})
}
}
@@ -41,6 +41,8 @@ impl WorkspaceHttpObjectiveBackend {
Ok(ToolOutput {
summary: format!("Listed {count} objective(s)"),
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
attachments: Vec::new(),
})
}
@@ -249,6 +251,8 @@ fn objective_output(summary: String, response: ObjectiveDetail) -> Result<ToolOu
Ok(ToolOutput {
summary,
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
attachments: Vec::new(),
})
}
@@ -384,6 +384,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -111,6 +111,8 @@ impl Tool for TaskListTool {
Ok(ToolOutput {
summary: list_overview(active_tasks.len(), tasks.len()),
content: Some(render_task_list(&tasks)),
attachments: Vec::new(),
})
}
}
@@ -131,6 +133,8 @@ impl Tool for TaskGetTool {
Ok(ToolOutput {
summary: format!("Task {} ({}) {}", task.taskid, task.status, task.subject),
content: Some(content),
attachments: Vec::new(),
})
}
}
@@ -170,6 +174,8 @@ fn task_output(summary: String, task: &TaskEntry) -> ToolOutput {
ToolOutput {
summary,
content: Some(serde_json::to_string_pretty(task).unwrap_or_default()),
attachments: Vec::new(),
}
}
@@ -739,6 +739,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
+5
View File
@@ -781,6 +781,7 @@ fn render_list_resources_result(result: ListResourcesResult) -> Result<ToolOutpu
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -822,6 +823,7 @@ fn render_read_resource_result(result: ReadResourceResult) -> Result<ToolOutput,
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -876,6 +878,7 @@ fn render_list_prompts_result(result: ListPromptsResult) -> Result<ToolOutput, T
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -924,6 +927,7 @@ fn render_get_prompt_result(result: GetPromptResult) -> Result<ToolOutput, ToolE
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
@@ -1242,6 +1246,7 @@ fn render_call_tool_result(
Ok(ToolOutput {
summary,
content: Some(content),
attachments: Vec::new(),
})
}
+7 -1
View File
@@ -4098,6 +4098,8 @@ impl PluginInstance {
Ok(ToolOutput {
summary: format!("{tool_name}: {tool_calls}"),
content: Some(String::from_utf8_lossy(&input).to_string()),
attachments: Vec::new(),
})
}
PluginInstanceRuntime::ComponentInstance(runtime) => {
@@ -5447,7 +5449,11 @@ fn decode_plugin_wasm_output(bytes: &[u8]) -> Result<ToolOutput, PluginWasmError
));
}
};
Ok(ToolOutput { summary, content })
Ok(ToolOutput {
summary,
content,
attachments: Vec::new(),
})
}
fn bounded_message(message: impl Into<String>) -> String {
+4
View File
@@ -328,6 +328,8 @@ impl Interceptor for WorkerInterceptor {
output: ToolOutput {
summary: info.result.summary.clone(),
content: info.result.content.clone(),
attachments: Vec::new(),
},
};
for hook in &self.registry.post_tool_call {
@@ -911,6 +913,8 @@ mod tests {
ToolOutput {
summary: "ok".into(),
content: Some("full".into()),
attachments: Vec::new(),
},
),
meta: info.meta,
+2
View File
@@ -89,6 +89,8 @@ mod tests {
output: ToolOutput {
summary: "result".to_string(),
content: None,
attachments: Vec::new(),
},
}
}
+3
View File
@@ -69,6 +69,7 @@ impl Tool for SubWorkerListTool {
Ok(ToolOutput {
summary: format!("listed {count} child SubWorker(s)"),
content: Some(content),
attachments: Vec::new(),
})
}
}
@@ -124,6 +125,7 @@ impl Tool for SubWorkerSendTool {
return Ok(ToolOutput {
summary: format!("sent message to `{}`", input.name),
content: None,
attachments: Vec::new(),
});
}
Err(unknown_worker_err(&input.name))
@@ -177,6 +179,7 @@ impl Tool for SubWorkerStopTool {
input.name
),
content: None,
attachments: Vec::new(),
});
}
Err(unknown_worker_err(&input.name))
+2
View File
@@ -492,6 +492,8 @@ impl Tool for SubWorkerSpawnTool {
Ok(ToolOutput {
summary: format!("spawned internal worker `{}`", input.name),
content: None,
attachments: Vec::new(),
})
}
}