feat: make image attachments request-local
This commit is contained in:
@@ -1790,6 +1790,19 @@ impl<C: LlmClient, S: EngineState, A: Send + Sync> Engine<C, S, A> {
|
||||
// this clone, so the caller-owned `history` stays intact.
|
||||
let mut request_context = history.items_cloned();
|
||||
|
||||
// Binary tool attachments are single-consumption request details. Keep fresh
|
||||
// attachments emitted after the latest model output for the immediate follow-up,
|
||||
// but stop resending them once a model response has consumed them. Durable history
|
||||
// remains untouched and retains the tool summary needed to fetch the file again.
|
||||
let projected_attachments =
|
||||
crate::prune::project_consumed_attachments(&mut request_context);
|
||||
if projected_attachments > 0 {
|
||||
debug!(
|
||||
projected_attachments,
|
||||
"Projected consumed tool-result attachments out of request context"
|
||||
);
|
||||
}
|
||||
|
||||
// Prune projection: if both the config and the savings
|
||||
// estimator are configured, drop ToolResult.content from
|
||||
// prunable candidates whose estimated savings meet the
|
||||
|
||||
@@ -133,6 +133,33 @@ pub fn project(items: &mut [Item], indices: &[usize]) -> usize {
|
||||
count
|
||||
}
|
||||
|
||||
/// Remove binary attachments after the first model request that could consume them.
|
||||
///
|
||||
/// Attachments on tool results following the latest model output are fresh outputs for the
|
||||
/// immediate follow-up request and remain visible. Once a later assistant/reasoning/tool-call
|
||||
/// item exists, the model has already had an opportunity to inspect the attachment. Subsequent
|
||||
/// requests keep the durable tool summary and can call the originating tool again when the image
|
||||
/// is still needed. Only the request-context clone is modified.
|
||||
pub fn project_consumed_attachments(items: &mut [Item]) -> usize {
|
||||
let latest_model_output = items.iter().rposition(|item| {
|
||||
item.is_assistant_message() || item.is_reasoning() || item.is_tool_call()
|
||||
});
|
||||
let mut count = 0;
|
||||
for (index, item) in items.iter_mut().enumerate() {
|
||||
let is_fresh = latest_model_output.is_some_and(|boundary| index > boundary);
|
||||
if is_fresh {
|
||||
continue;
|
||||
}
|
||||
if let Item::ToolResult { attachments, .. } = item
|
||||
&& !attachments.is_empty()
|
||||
{
|
||||
attachments.clear();
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
count
|
||||
}
|
||||
|
||||
/// Indices of detailed `Item::ToolResult` values that lie before
|
||||
/// the suffix protected by `protected_tokens`. Pure: does not mutate `items`.
|
||||
///
|
||||
@@ -412,6 +439,52 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
fn image_result(call_id: &str, bytes: &[u8]) -> Item {
|
||||
Item::tool_result_item_with_attachments(
|
||||
call_id,
|
||||
"Attached image.png",
|
||||
None,
|
||||
false,
|
||||
vec![crate::tool::Attachment::Image(
|
||||
crate::tool::ImageAttachment::new("image/png", bytes.to_vec()),
|
||||
)],
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consumed_attachment_projection_keeps_only_fresh_tool_outputs() {
|
||||
let mut immediate = vec![
|
||||
Item::tool_call("image-1", "ViewImage", r#"{"path":"first.png"}"#),
|
||||
image_result("image-1", b"first"),
|
||||
];
|
||||
assert_eq!(project_consumed_attachments(&mut immediate), 0);
|
||||
assert!(matches!(
|
||||
&immediate[1],
|
||||
Item::ToolResult { attachments, .. } if attachments.len() == 1
|
||||
));
|
||||
|
||||
let mut consumed = immediate.clone();
|
||||
consumed.push(Item::assistant_message("I inspected the image."));
|
||||
consumed.push(Item::user_message("continue"));
|
||||
assert_eq!(project_consumed_attachments(&mut consumed), 1);
|
||||
assert!(matches!(
|
||||
&consumed[1],
|
||||
Item::ToolResult { attachments, .. } if attachments.is_empty()
|
||||
));
|
||||
|
||||
consumed.push(Item::tool_call(
|
||||
"image-2",
|
||||
"ViewImage",
|
||||
r#"{"path":"second.png"}"#,
|
||||
));
|
||||
consumed.push(image_result("image-2", b"second"));
|
||||
assert_eq!(project_consumed_attachments(&mut consumed), 0);
|
||||
assert!(matches!(
|
||||
consumed.last(),
|
||||
Some(Item::ToolResult { attachments, .. }) if attachments.len() == 1
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn project_skips_already_pruned_items() {
|
||||
// indices points at an item whose content is already None.
|
||||
|
||||
@@ -47,11 +47,18 @@ pub fn prefix_bytes(items: &[Item]) -> Vec<u64> {
|
||||
prefix
|
||||
}
|
||||
|
||||
/// 1 Item の大きさ。JSON シリアライズ長を使う粗い近似。
|
||||
/// トークン数との絶対変換ではなく区間の按分にしか使わないので、
|
||||
/// プロバイダごとの overhead は比率でキャンセルされる。
|
||||
/// 1 Item の text-shaped size。JSON シリアライズ長を使う粗い近似。
|
||||
///
|
||||
/// Binary attachments are deliberately projected out. Their durable base64 encoding is a
|
||||
/// storage representation, not provider text, so charging it at the text `bytes / 4` fallback
|
||||
/// rate produces meaningless estimates. Exact provider usage measurements remain authoritative
|
||||
/// when an attachment was actually sent.
|
||||
pub fn item_bytes(item: &Item) -> u64 {
|
||||
serde_json::to_string(item)
|
||||
let mut projected = item.clone();
|
||||
if let Item::ToolResult { attachments, .. } = &mut projected {
|
||||
attachments.clear();
|
||||
}
|
||||
serde_json::to_string(&projected)
|
||||
.map(|s| s.len() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
@@ -265,6 +272,31 @@ mod tests {
|
||||
assert_eq!(est.tokens, 30_000 + delta_bytes / 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_does_not_count_durable_image_bytes_as_text_tokens() {
|
||||
let small = Item::tool_result_item_with_attachments(
|
||||
"call",
|
||||
"Attached image.png",
|
||||
None,
|
||||
false,
|
||||
vec![crate::tool::Attachment::Image(
|
||||
crate::tool::ImageAttachment::new("image/png", vec![1_u8; 8]),
|
||||
)],
|
||||
);
|
||||
let large = Item::tool_result_item_with_attachments(
|
||||
"call",
|
||||
"Attached image.png",
|
||||
None,
|
||||
false,
|
||||
vec![crate::tool::Attachment::Image(
|
||||
crate::tool::ImageAttachment::new("image/png", vec![1_u8; 1_000_000]),
|
||||
)],
|
||||
);
|
||||
|
||||
assert_eq!(item_bytes(&small), item_bytes(&large));
|
||||
assert_eq!(total_tokens(&[small], &[]), total_tokens(&[large], &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn total_zero_history_is_zero() {
|
||||
let est = total_tokens(&[], &[]);
|
||||
|
||||
Reference in New Issue
Block a user