feat: add scoped multimodal image attachments
This commit is contained in:
@@ -228,6 +228,8 @@ impl Tool for SearchSessionLogTool {
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: (!content.is_empty()).then_some(content),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -290,6 +292,8 @@ impl Tool for ReadSessionItemsTool {
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: (!content.is_empty()).then_some(content),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -395,6 +399,8 @@ impl Tool for MarkReadRequiredTool {
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: None,
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -423,6 +429,8 @@ impl Tool for AddReferenceTool {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("Added reference {}", params.file_path.display()),
|
||||
content: None,
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -452,6 +460,8 @@ impl Tool for WriteSummaryTool {
|
||||
Ok(ToolOutput {
|
||||
summary: note.to_string(),
|
||||
content: None,
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -648,6 +648,12 @@ where
|
||||
tracker.clone(),
|
||||
bash_output_dir,
|
||||
));
|
||||
if feature_config.image.enabled && model_supports_image_attachments(&spawner_manifest.model)
|
||||
{
|
||||
worker
|
||||
.engine_mut()
|
||||
.register_tool(tools::view_image_tool(workdir.clone()));
|
||||
}
|
||||
(Some(workdir), Some(tracker))
|
||||
} else {
|
||||
(None, None)
|
||||
@@ -1570,6 +1576,16 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn model_supports_image_attachments(model: &manifest::ModelManifest) -> bool {
|
||||
manifest::model_catalog::resolve_model_manifest(model).is_ok_and(|model| {
|
||||
model.capability.is_some_and(|capability| capability.vision)
|
||||
&& matches!(
|
||||
model.scheme,
|
||||
manifest::SchemeKind::OpenaiChat | manifest::SchemeKind::OpenaiResponses
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn build_greeting<C, St>(worker: &Worker<C, St>) -> protocol::Greeting
|
||||
where
|
||||
C: LlmClient,
|
||||
@@ -1649,6 +1665,20 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::net::UnixListener;
|
||||
|
||||
#[test]
|
||||
fn image_attachment_gate_requires_vision_and_supported_openai_scheme() {
|
||||
let openai = manifest::ModelManifest {
|
||||
ref_: Some("codex-oauth/gpt-5.6-sol".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let anthropic = manifest::ModelManifest {
|
||||
ref_: Some("anthropic/claude-opus-4-8".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(model_supports_image_attachments(&openai));
|
||||
assert!(!model_supports_image_attachments(&anthropic));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_run_parent_origin_table() {
|
||||
assert!(PendingRun::Run(Vec::new()).is_parent_originated());
|
||||
|
||||
@@ -854,6 +854,7 @@ where
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(json_content(&items)?),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -890,6 +891,8 @@ where
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(json_content(&result)?),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -982,6 +985,7 @@ where
|
||||
Ok(ToolOutput {
|
||||
summary: format!("sent peer message to `{}`", input.name),
|
||||
content: None,
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -253,6 +253,7 @@ impl Tool for RequestFlowTransitionTool {
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -422,6 +423,7 @@ impl Tool for FinishFlowVerificationTool {
|
||||
Ok(ToolOutput {
|
||||
summary: "Recorded complete Flow verification verdicts.".to_string(),
|
||||
content: Some("{\"accepted\":true}".to_string()),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -492,6 +492,7 @@ fn workdir_output<T: Serialize>(summary: String, value: &T) -> Result<ToolOutput
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(serde_json::to_string_pretty(value).map_err(decode_error)?),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -283,6 +283,7 @@ impl Tool for WorkspaceWorkerTool {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("{} completed", self.operation.tool_name()),
|
||||
content: Some(response.body),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -292,6 +292,7 @@ fn tool_output(output: MemoryToolOutput) -> ToolOutput {
|
||||
ToolOutput {
|
||||
summary: output.summary,
|
||||
content: output.content,
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -235,6 +235,7 @@ impl Tool for StageMemoryCandidateTool {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("Staged Memory candidate {staging_id}."),
|
||||
content: Some(format!("staging_id: {staging_id}")),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -281,6 +282,7 @@ impl Tool for FinishMemoryExtractionTool {
|
||||
format!("Finished extraction with {actual} staged candidate(s).")
|
||||
}),
|
||||
content: None,
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,8 @@ impl WorkspaceHttpObjectiveBackend {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("Listed {count} objective(s)"),
|
||||
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -249,6 +251,8 @@ fn objective_output(summary: String, response: ObjectiveDetail) -> Result<ToolOu
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(serde_json::to_string_pretty(&response).map_err(decode_error)?),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -384,6 +384,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -111,6 +111,8 @@ impl Tool for TaskListTool {
|
||||
Ok(ToolOutput {
|
||||
summary: list_overview(active_tasks.len(), tasks.len()),
|
||||
content: Some(render_task_list(&tasks)),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -131,6 +133,8 @@ impl Tool for TaskGetTool {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("Task {} ({}) {}", task.taskid, task.status, task.subject),
|
||||
content: Some(content),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -170,6 +174,8 @@ fn task_output(summary: String, task: &TaskEntry) -> ToolOutput {
|
||||
ToolOutput {
|
||||
summary,
|
||||
content: Some(serde_json::to_string_pretty(task).unwrap_or_default()),
|
||||
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -739,6 +739,7 @@ fn json_output(summary: String, value: serde_json::Value) -> Result<ToolOutput,
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -781,6 +781,7 @@ fn render_list_resources_result(result: ListResourcesResult) -> Result<ToolOutpu
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -822,6 +823,7 @@ fn render_read_resource_result(result: ReadResourceResult) -> Result<ToolOutput,
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -876,6 +878,7 @@ fn render_list_prompts_result(result: ListPromptsResult) -> Result<ToolOutput, T
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -924,6 +927,7 @@ fn render_get_prompt_result(result: GetPromptResult) -> Result<ToolOutput, ToolE
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1242,6 +1246,7 @@ fn render_call_tool_result(
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -4098,6 +4098,8 @@ impl PluginInstance {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("{tool_name}: {tool_calls}"),
|
||||
content: Some(String::from_utf8_lossy(&input).to_string()),
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
PluginInstanceRuntime::ComponentInstance(runtime) => {
|
||||
@@ -5447,7 +5449,11 @@ fn decode_plugin_wasm_output(bytes: &[u8]) -> Result<ToolOutput, PluginWasmError
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok(ToolOutput { summary, content })
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content,
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn bounded_message(message: impl Into<String>) -> String {
|
||||
|
||||
@@ -328,6 +328,8 @@ impl Interceptor for WorkerInterceptor {
|
||||
output: ToolOutput {
|
||||
summary: info.result.summary.clone(),
|
||||
content: info.result.content.clone(),
|
||||
|
||||
attachments: Vec::new(),
|
||||
},
|
||||
};
|
||||
for hook in &self.registry.post_tool_call {
|
||||
@@ -911,6 +913,8 @@ mod tests {
|
||||
ToolOutput {
|
||||
summary: "ok".into(),
|
||||
content: Some("full".into()),
|
||||
|
||||
attachments: Vec::new(),
|
||||
},
|
||||
),
|
||||
meta: info.meta,
|
||||
|
||||
@@ -89,6 +89,8 @@ mod tests {
|
||||
output: ToolOutput {
|
||||
summary: "result".to_string(),
|
||||
content: None,
|
||||
|
||||
attachments: Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,6 +69,7 @@ impl Tool for SubWorkerListTool {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("listed {count} child SubWorker(s)"),
|
||||
content: Some(content),
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -124,6 +125,7 @@ impl Tool for SubWorkerSendTool {
|
||||
return Ok(ToolOutput {
|
||||
summary: format!("sent message to `{}`", input.name),
|
||||
content: None,
|
||||
attachments: Vec::new(),
|
||||
});
|
||||
}
|
||||
Err(unknown_worker_err(&input.name))
|
||||
@@ -177,6 +179,7 @@ impl Tool for SubWorkerStopTool {
|
||||
input.name
|
||||
),
|
||||
content: None,
|
||||
attachments: Vec::new(),
|
||||
});
|
||||
}
|
||||
Err(unknown_worker_err(&input.name))
|
||||
|
||||
@@ -492,6 +492,8 @@ impl Tool for SubWorkerSpawnTool {
|
||||
Ok(ToolOutput {
|
||||
summary: format!("spawned internal worker `{}`", input.name),
|
||||
content: None,
|
||||
|
||||
attachments: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user