refactor: rename pod crate to worker
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
//! Sync buffer for `session_metrics::Metric` values queued from inside
|
||||
//! Engine callbacks (which run synchronously and cannot themselves
|
||||
//! perform `async` store writes).
|
||||
//!
|
||||
//! Worker drains this buffer in `persist_turn` and writes each metric via
|
||||
//! `session_metrics::record_metric`, alongside the regular `LlmUsage`
|
||||
//! entries.
|
||||
|
||||
use std::sync::Mutex;
|
||||
|
||||
use session_metrics::Metric;
|
||||
|
||||
pub(crate) struct MetricsTracker {
|
||||
pending: Mutex<Vec<Metric>>,
|
||||
}
|
||||
|
||||
impl MetricsTracker {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
pending: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue a metric for the next `persist_turn` flush.
|
||||
pub(crate) fn push(&self, metric: Metric) {
|
||||
self.pending.lock().unwrap().push(metric);
|
||||
}
|
||||
|
||||
/// Drain all queued metrics. Called by Worker after a run completes.
|
||||
pub(crate) fn drain(&self) -> Vec<Metric> {
|
||||
std::mem::take(&mut *self.pending.lock().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn push_then_drain_returns_in_order_and_clears() {
|
||||
let t = MetricsTracker::new();
|
||||
t.push(Metric::now("a"));
|
||||
t.push(Metric::now("b"));
|
||||
let drained = t.drain();
|
||||
assert_eq!(drained.len(), 2);
|
||||
assert_eq!(drained[0].name, "a");
|
||||
assert_eq!(drained[1].name, "b");
|
||||
assert!(t.drain().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
pub(crate) mod metrics_tracker;
|
||||
pub(crate) mod prune;
|
||||
pub(crate) mod state;
|
||||
pub(crate) mod token_counter;
|
||||
pub(crate) mod usage_tracker;
|
||||
pub(crate) mod worker;
|
||||
@@ -0,0 +1,124 @@
|
||||
//! Prune integration — wires the Engine's prune projection to the Worker's
|
||||
//! usage-history-backed token accounting.
|
||||
//!
|
||||
//! Engine 自身がコンテキスト射影を行う(`worker.rs` の `request_context` 構築
|
||||
//! 直後)。Engine は usage 履歴を知らないので、`min_savings` 判定に使う savings
|
||||
//! の見積もりはコールバックで外部から注入する。このモジュールはそのコールバック
|
||||
//! を組み立てて Engine に差し込むための `impl Worker` を提供する。
|
||||
//!
|
||||
//! 同じ経路で `PruneObserver` も install し、評価のたびに `prune.fire` /
|
||||
//! `prune.skip` metric を `MetricsTracker` に積む。`Fired` 時は uuid を
|
||||
//! `UsageTracker` にも stash しておき、後続の `LlmUsage` と組で
|
||||
//! `prune.post_request` を吐けるようにする。
|
||||
|
||||
use llm_engine::Item;
|
||||
use llm_engine::llm_client::client::LlmClient;
|
||||
use llm_engine::prune::{
|
||||
PruneConfig, PruneDecision, PruneObserver, SavingsEstimator, TokenEstimator,
|
||||
};
|
||||
use session_metrics::Metric;
|
||||
use session_store::Store;
|
||||
|
||||
use crate::Worker;
|
||||
use crate::compact::token_counter::{
|
||||
EstimateSource, savings_for_prune_impl, token_estimates_for_prune_impl,
|
||||
};
|
||||
|
||||
impl<C: LlmClient, St: Store> Worker<C, St> {
|
||||
/// Enable prune projection on the underlying Engine.
|
||||
///
|
||||
/// Registers the config and token/savings-estimator closures on the Engine.
|
||||
/// The estimators combine persisted [`Worker::usage_history_handle`] records
|
||||
/// with in-flight `UsageTracker` records so multi-request tool loops can
|
||||
/// prune before the surrounding Worker run finishes.
|
||||
///
|
||||
/// Measurement-less estimates (before the first LLM call, or immediately
|
||||
/// after a compact) return `0` from the estimator, which naturally
|
||||
/// prevents the prune projection from firing until usage data exists.
|
||||
///
|
||||
/// Also installs a [`PruneObserver`] that pushes `prune.fire` /
|
||||
/// `prune.skip` metrics into the shared [`MetricsTracker`]. On `Fired`
|
||||
/// the observer additionally stashes a fresh correlation_id in
|
||||
/// [`UsageTracker`] so the next `LlmUsage` can be paired with a
|
||||
/// `prune.post_request` metric carrying the same id.
|
||||
pub fn attach_prune(&mut self, config: PruneConfig) {
|
||||
let usage_history_for_tokens = self.usage_history_handle();
|
||||
let usage_tracker_for_tokens = self.usage_tracker_handle();
|
||||
let token_estimator: TokenEstimator = Box::new(move |history: &[Item]| {
|
||||
let mut snapshot = usage_history_for_tokens
|
||||
.lock()
|
||||
.expect("usage_history poisoned")
|
||||
.clone();
|
||||
snapshot.extend(usage_tracker_for_tokens.records());
|
||||
token_estimates_for_prune_impl(history, &snapshot)
|
||||
});
|
||||
|
||||
let usage_history_for_savings = self.usage_history_handle();
|
||||
let usage_tracker_for_savings = self.usage_tracker_handle();
|
||||
let estimator: SavingsEstimator = Box::new(move |history: &[Item], indices| {
|
||||
let mut snapshot = usage_history_for_savings
|
||||
.lock()
|
||||
.expect("usage_history poisoned")
|
||||
.clone();
|
||||
snapshot.extend(usage_tracker_for_savings.records());
|
||||
let est = savings_for_prune_impl(history, &snapshot, indices);
|
||||
match est.source {
|
||||
EstimateSource::NoData => 0,
|
||||
_ => est.tokens,
|
||||
}
|
||||
});
|
||||
|
||||
let metrics = self.metrics_tracker_handle();
|
||||
let usage_tracker = self.usage_tracker_handle();
|
||||
let observer: PruneObserver = Box::new(move |eval| match &eval.decision {
|
||||
PruneDecision::Fired { .. } => {
|
||||
let correlation_id = uuid::Uuid::now_v7().to_string();
|
||||
let mut metric = Metric::now("prune.fire")
|
||||
.with_value(eval.estimated_savings as f64)
|
||||
.with_correlation_id(&correlation_id)
|
||||
.with_dimension("candidate_count", eval.candidate_count.to_string());
|
||||
if let Some(protected_start) = eval.protected_start_index {
|
||||
metric =
|
||||
metric.with_dimension("protected_start_index", protected_start.to_string());
|
||||
}
|
||||
metrics.push(metric);
|
||||
usage_tracker.note_correlation_id(correlation_id);
|
||||
}
|
||||
PruneDecision::SkippedNoCandidates => {
|
||||
metrics.push(Metric::now("prune.skip").with_dimension("reason", "no_candidates"));
|
||||
}
|
||||
PruneDecision::SkippedBelowMinSavings => {
|
||||
let mut metric = Metric::now("prune.skip")
|
||||
.with_dimension("reason", "below_min_savings")
|
||||
.with_dimension("candidate_count", eval.candidate_count.to_string())
|
||||
.with_value(eval.estimated_savings as f64);
|
||||
if let Some(protected_start) = eval.protected_start_index {
|
||||
metric =
|
||||
metric.with_dimension("protected_start_index", protected_start.to_string());
|
||||
}
|
||||
metrics.push(metric);
|
||||
}
|
||||
});
|
||||
|
||||
let worker = self.engine_mut();
|
||||
worker.set_prune_config(Some(config));
|
||||
worker.set_token_estimator(Some(token_estimator));
|
||||
worker.set_savings_estimator(Some(estimator));
|
||||
worker.set_prune_observer(Some(observer));
|
||||
}
|
||||
|
||||
/// If the manifest has a `[compaction]` section, build a `PruneConfig`
|
||||
/// from its `prune_*` fields and call [`attach_prune`](Self::attach_prune).
|
||||
/// Otherwise no-op. Called from all Worker constructors so prune is
|
||||
/// active whenever the manifest asks for it.
|
||||
pub(crate) fn apply_prune_from_manifest(&mut self) {
|
||||
let Some(compaction) = self.manifest().compaction.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let config = PruneConfig {
|
||||
protected_tokens: compaction.prune_protected_tokens,
|
||||
min_savings: compaction.prune_min_savings,
|
||||
};
|
||||
self.attach_prune(config);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
//! Shared state for compaction decisions.
|
||||
//!
|
||||
//! Holds the two configured thresholds and circuit-breaker / thrash-detection
|
||||
//! flags shared between:
|
||||
//! - `WorkerInterceptor` (reads `request_threshold` — the *safety net* for
|
||||
//! between-requests yielding)
|
||||
//! - `Worker::try_pre_run_compact` (reads `post_run_threshold` — the
|
||||
//! *proactive* check before the next turn starts)
|
||||
//! - `Worker::run()` / `resume()` (circuit breaker, thrash detection)
|
||||
//!
|
||||
//! Current occupancy (input-token count) is **not** stored here. The single
|
||||
//! source of truth is `session_store::UsageRecord` (persisted per LLM call)
|
||||
//! projected through `Worker::total_tokens()`. Callers pass the current
|
||||
//! occupancy to `exceeds_*` at check time.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
const MAX_COMPACT_FAILURES: usize = 3;
|
||||
|
||||
/// Shared mutable state for compaction decisions.
|
||||
pub(crate) struct CompactState {
|
||||
/// Between-turns threshold (proactive). Checked before the next turn
|
||||
/// starts. `None` disables the pre-run check.
|
||||
post_run_threshold: Option<u64>,
|
||||
/// Between-requests threshold (safety net). Checked inside a turn
|
||||
/// before each LLM request. `None` disables the request check.
|
||||
request_threshold: Option<u64>,
|
||||
/// Token budget retained verbatim at the tail after compaction.
|
||||
retained_tokens: u64,
|
||||
/// Consecutive compact failures. At `MAX_COMPACT_FAILURES`, compaction is disabled.
|
||||
consecutive_failures: AtomicUsize,
|
||||
/// `true` immediately after a successful compact, cleared on next normal completion.
|
||||
just_compacted: AtomicBool,
|
||||
/// `true` when circuit breaker has tripped.
|
||||
disabled: AtomicBool,
|
||||
}
|
||||
|
||||
impl CompactState {
|
||||
pub(crate) fn new(
|
||||
post_run_threshold: Option<u64>,
|
||||
request_threshold: Option<u64>,
|
||||
retained_tokens: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
post_run_threshold,
|
||||
request_threshold,
|
||||
retained_tokens,
|
||||
consecutive_failures: AtomicUsize::new(0),
|
||||
just_compacted: AtomicBool::new(false),
|
||||
disabled: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Configured between-requests threshold (if any).
|
||||
pub(crate) fn request_threshold(&self) -> Option<u64> {
|
||||
self.request_threshold
|
||||
}
|
||||
|
||||
/// Token budget retained verbatim at the tail after compaction.
|
||||
pub(crate) fn retained_tokens(&self) -> u64 {
|
||||
self.retained_tokens
|
||||
}
|
||||
|
||||
/// Whether compaction has been disabled by the circuit breaker.
|
||||
pub(crate) fn is_disabled(&self) -> bool {
|
||||
self.disabled.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Whether `current_tokens` exceeds the between-requests threshold.
|
||||
/// Returns `false` when `request_threshold` is unset.
|
||||
pub(crate) fn exceeds_request(&self, current_tokens: u64) -> bool {
|
||||
self.request_threshold
|
||||
.map(|t| current_tokens > t)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether `current_tokens` exceeds the post-run threshold.
|
||||
/// Returns `false` when `post_run_threshold` is unset.
|
||||
pub(crate) fn exceeds_post_run(&self, current_tokens: u64) -> bool {
|
||||
self.post_run_threshold
|
||||
.map(|t| current_tokens > t)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether a compact just completed (for thrash detection).
|
||||
pub(crate) fn just_compacted(&self) -> bool {
|
||||
self.just_compacted.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Set or clear the just_compacted flag.
|
||||
pub(crate) fn set_just_compacted(&self, val: bool) {
|
||||
self.just_compacted.store(val, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record a successful compaction: reset failure counter, set just_compacted.
|
||||
pub(crate) fn record_compact_success(&self) {
|
||||
self.consecutive_failures.store(0, Ordering::Relaxed);
|
||||
self.just_compacted.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record a compaction failure. Disables compaction after MAX_COMPACT_FAILURES.
|
||||
pub(crate) fn record_compact_failure(&self) {
|
||||
let prev = self.consecutive_failures.fetch_add(1, Ordering::Relaxed);
|
||||
if prev + 1 >= MAX_COMPACT_FAILURES {
|
||||
self.disabled.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn both_thresholds_configured() {
|
||||
let state = CompactState::new(Some(80_000), Some(90_000), 8_000);
|
||||
assert_eq!(state.request_threshold(), Some(90_000));
|
||||
assert_eq!(state.retained_tokens(), 8_000);
|
||||
|
||||
assert!(!state.exceeds_request(70_000));
|
||||
assert!(!state.exceeds_post_run(70_000));
|
||||
|
||||
assert!(!state.exceeds_request(85_000));
|
||||
assert!(state.exceeds_post_run(85_000));
|
||||
|
||||
assert!(state.exceeds_request(95_000));
|
||||
assert!(state.exceeds_post_run(95_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn post_run_only() {
|
||||
let state = CompactState::new(Some(80_000), None, 8_000);
|
||||
// request check always false when threshold is None.
|
||||
assert!(!state.exceeds_request(1_000_000));
|
||||
assert!(state.exceeds_post_run(85_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_only() {
|
||||
let state = CompactState::new(None, Some(90_000), 8_000);
|
||||
assert!(!state.exceeds_post_run(1_000_000));
|
||||
assert!(state.exceeds_request(95_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_none_disables_all_checks() {
|
||||
let state = CompactState::new(None, None, 8_000);
|
||||
assert!(!state.exceeds_request(1_000_000));
|
||||
assert!(!state.exceeds_post_run(1_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_breaker_trips_after_max_failures() {
|
||||
let state = CompactState::new(Some(80_000), Some(90_000), 8_000);
|
||||
assert!(!state.is_disabled());
|
||||
|
||||
state.record_compact_failure();
|
||||
assert!(!state.is_disabled());
|
||||
state.record_compact_failure();
|
||||
assert!(!state.is_disabled());
|
||||
state.record_compact_failure();
|
||||
assert!(state.is_disabled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_resets_failure_count() {
|
||||
let state = CompactState::new(Some(80_000), Some(90_000), 8_000);
|
||||
state.record_compact_failure();
|
||||
state.record_compact_failure();
|
||||
assert!(!state.is_disabled());
|
||||
|
||||
state.record_compact_success();
|
||||
assert!(state.just_compacted());
|
||||
|
||||
state.record_compact_failure();
|
||||
state.record_compact_failure();
|
||||
assert!(!state.is_disabled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn just_compacted_lifecycle() {
|
||||
let state = CompactState::new(Some(80_000), Some(90_000), 8_000);
|
||||
assert!(!state.just_compacted());
|
||||
|
||||
state.record_compact_success();
|
||||
assert!(state.just_compacted());
|
||||
|
||||
state.set_just_compacted(false);
|
||||
assert!(!state.just_compacted());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,557 @@
|
||||
//! Compact / prune 専用のトークン会計補助。
|
||||
//!
|
||||
//! 汎用部分(`prefix_bytes`, `tokens_at`, `total_tokens`, `total_tokens_at`)は
|
||||
//! [`llm_engine::token_counter`] にあり、`UsageRecord` の列と現在の history から
|
||||
//! pure に推定する。本モジュールは compact / prune 固有のロジック
|
||||
//! (`split_for_retained`, `savings_for_prune`)と、Worker 上の公開 API に
|
||||
//! 限定する。
|
||||
//!
|
||||
//! # 方針
|
||||
//!
|
||||
//! - ローカルトークナイザは持たない。実測値があればそれを採用し、
|
||||
//! measurement 間はバイト数で按分、最新 measurement より先は byte/4 で外挿する
|
||||
//! - Compact の retained split では、request-time pruning / projection 後の
|
||||
//! `UsageRecord` を persisted history prefix の単調系列として扱わない。
|
||||
//! 現在の prompt occupancy 推定を raw serialized bytes に配分し、末尾の
|
||||
//! persisted tail サイズで cut を決める。
|
||||
//! - 推定の出どころは [`EstimateSource`] で呼び出し側に明示する。
|
||||
//! 課金判断には使えないが、compact / prune の閾値判定には十分な精度
|
||||
|
||||
use llm_engine::llm_client::client::LlmClient;
|
||||
use llm_engine::token_counter::{item_bytes, prefix_bytes, tokens_at};
|
||||
use llm_engine::{Item, UsageRecord};
|
||||
use session_store::Store;
|
||||
|
||||
pub use llm_engine::token_counter::{EstimateSource, TokenEstimate};
|
||||
|
||||
use crate::Worker;
|
||||
|
||||
/// history を分割する位置。
|
||||
///
|
||||
/// `items[..index]` が捨てる/要約される側、`items[index..]` が残る側。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SplitPoint {
|
||||
pub index: usize,
|
||||
pub source: EstimateSource,
|
||||
}
|
||||
|
||||
fn split_for_retained_impl(history: &[Item], records: &[UsageRecord], retained: u64) -> SplitPoint {
|
||||
let prefix = prefix_bytes(history);
|
||||
let current = tokens_at(history, records, history.len(), &prefix);
|
||||
if current.tokens <= retained {
|
||||
return SplitPoint {
|
||||
index: 0,
|
||||
source: current.source,
|
||||
};
|
||||
}
|
||||
|
||||
let cut_index = split_index_by_retained_bytes(&prefix, current.tokens, retained);
|
||||
SplitPoint {
|
||||
index: balance_to_pair_boundary(history, cut_index),
|
||||
source: current.source,
|
||||
}
|
||||
}
|
||||
|
||||
fn split_index_by_retained_bytes(prefix: &[u64], total_tokens: u64, retained_tokens: u64) -> usize {
|
||||
debug_assert!(!prefix.is_empty());
|
||||
|
||||
let len = prefix.len() - 1;
|
||||
if len == 0 {
|
||||
return 0;
|
||||
}
|
||||
if retained_tokens == 0 {
|
||||
return len;
|
||||
}
|
||||
|
||||
let total_bytes = *prefix.last().unwrap_or(&0);
|
||||
if total_bytes == 0 || total_tokens == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let raw_fallback_tokens = ceil_div_u128(total_bytes as u128, 4) as u64;
|
||||
let rate_tokens = total_tokens.max(raw_fallback_tokens);
|
||||
let target_retained_bytes = ceil_div_u128(
|
||||
retained_tokens as u128 * total_bytes as u128,
|
||||
rate_tokens as u128,
|
||||
)
|
||||
.min(total_bytes as u128) as u64;
|
||||
|
||||
// Drop as many complete Items as possible while keeping the raw persisted
|
||||
// suffix at or above the retained budget. This is monotonic in serialized
|
||||
// history size and intentionally does not inspect per-history_len
|
||||
// UsageRecords: request-time usage can move up and down after pruning /
|
||||
// projection, so it is not a valid prefix series for retained split. The
|
||||
// byte/4 fallback is kept as a lower bound for raw persisted size so a
|
||||
// heavily-pruned request measurement cannot justify retaining megabytes of
|
||||
// history.
|
||||
let mut cut = 0;
|
||||
for (idx, bytes_before) in prefix.iter().enumerate().take(len + 1) {
|
||||
let suffix_bytes = total_bytes.saturating_sub(*bytes_before);
|
||||
if suffix_bytes >= target_retained_bytes {
|
||||
cut = idx;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
cut
|
||||
}
|
||||
|
||||
fn ceil_div_u128(n: u128, d: u128) -> u128 {
|
||||
debug_assert!(d > 0);
|
||||
if n == 0 { 0 } else { ((n - 1) / d) + 1 }
|
||||
}
|
||||
|
||||
/// `history[cut..]` が `ToolCall` / `ToolResult` のペア境界を尊重するよう
|
||||
/// `cut` を後退させる。
|
||||
///
|
||||
/// LLM API は「`ToolResult` を送るならその `ToolCall` も同じ request に
|
||||
/// 含まれていなければならない」というバリデーションを持つ。トークン数
|
||||
/// だけで切った `cut` は並列 tool 呼び出しの途中に落ちうるので、retained
|
||||
/// 側の先頭に対応 `ToolCall` を持たない `ToolResult`(orphan)が残ると
|
||||
/// 次セッション初回 request が API バリデーションで弾かれる。
|
||||
///
|
||||
/// 対策は「retained に入る `ToolResult` について、対応 `ToolCall` も
|
||||
/// retained に含まれる位置まで `cut` を引き下げる」こと。retained_tokens
|
||||
/// 予算は超えうるが、ここでは直接 LLM に投げる訳ではなく次の
|
||||
/// `pre_llm_request` で再評価されるだけなので safe。
|
||||
///
|
||||
/// アルゴリズム: history を末尾から走査し、retained 範囲内の `ToolResult`
|
||||
/// に出会うたびに対応 `ToolCall` の位置で `cut` を min 更新する。`cut` が
|
||||
/// 下がると以前は要約側だった位置が retained に入るので、後続走査で連鎖的
|
||||
/// に正しい位置まで引き下がる。`ToolCall` の `call_id` はユニークなので
|
||||
/// 事前にマップ化して O(n) で済ます。
|
||||
fn balance_to_pair_boundary(history: &[Item], cut: usize) -> usize {
|
||||
let mut idx = cut.min(history.len());
|
||||
if idx == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let call_positions: std::collections::HashMap<&str, usize> = history
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, item)| match item {
|
||||
Item::ToolCall { call_id, .. } => Some((call_id.as_str(), i)),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut k = history.len();
|
||||
while k > 0 {
|
||||
k -= 1;
|
||||
if k >= idx {
|
||||
if let Item::ToolResult { call_id, .. } = &history[k] {
|
||||
if let Some(&call_pos) = call_positions.get(call_id.as_str()) {
|
||||
if call_pos < idx {
|
||||
idx = call_pos;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
idx
|
||||
}
|
||||
|
||||
/// 1 つの ToolResult 項目について、`content` を `None` に射影したとき
|
||||
/// 減少するシリアライズ後バイト数。ToolResult 以外や既に content=None
|
||||
/// の item は 0 を返す。
|
||||
fn tool_result_content_bytes(item: &Item) -> u64 {
|
||||
if !matches!(
|
||||
item,
|
||||
Item::ToolResult {
|
||||
content: Some(_),
|
||||
..
|
||||
}
|
||||
) {
|
||||
return 0;
|
||||
}
|
||||
let mut cleared = item.clone();
|
||||
if let Item::ToolResult { content, .. } = &mut cleared {
|
||||
*content = None;
|
||||
}
|
||||
item_bytes(item).saturating_sub(item_bytes(&cleared))
|
||||
}
|
||||
|
||||
/// Prefix-boundary token estimates used by Prune to find its protected suffix.
|
||||
///
|
||||
/// Returns `history.len() + 1` entries where entry `i` estimates
|
||||
/// `history[..i]`. This shares the same [`tokens_at`] accounting as compact's
|
||||
/// retained-tail split and prune's savings estimate.
|
||||
pub(crate) fn token_estimates_for_prune_impl(
|
||||
history: &[Item],
|
||||
records: &[UsageRecord],
|
||||
) -> Vec<TokenEstimate> {
|
||||
let prefix = prefix_bytes(history);
|
||||
(0..=history.len())
|
||||
.map(|idx| tokens_at(history, records, idx, &prefix))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Prune 射影(`ToolResult.content = None`)で節約されるトークン数の推定。
|
||||
///
|
||||
/// `indices` は [`llm_engine::prune::prunable_indices`] が返す候補列を
|
||||
/// 想定する。各候補の content バイト差分を合算し、usage 履歴由来の
|
||||
/// tokens/byte レートでトークン数に換算する。範囲を「丸ごと drop」する
|
||||
/// のではなく、item 自体(summary 等)は残したままの値を返す点が
|
||||
/// `tokens_at` ベースの計算と異なる。
|
||||
pub(crate) fn savings_for_prune_impl(
|
||||
history: &[Item],
|
||||
records: &[UsageRecord],
|
||||
indices: &[usize],
|
||||
) -> TokenEstimate {
|
||||
let removed_bytes: u64 = indices
|
||||
.iter()
|
||||
.filter_map(|&i| history.get(i))
|
||||
.map(tool_result_content_bytes)
|
||||
.sum();
|
||||
|
||||
if removed_bytes == 0 {
|
||||
return TokenEstimate {
|
||||
tokens: 0,
|
||||
source: EstimateSource::Measured,
|
||||
};
|
||||
}
|
||||
|
||||
if records.is_empty() {
|
||||
return TokenEstimate {
|
||||
tokens: removed_bytes / 4,
|
||||
source: EstimateSource::NoData,
|
||||
};
|
||||
}
|
||||
|
||||
// 最新の measurement を使って tokens/byte を求め、バイト差分を換算する。
|
||||
// 実測値そのものではなく比率しか使わないので、history_len と
|
||||
// record.history_len が一致しなくても rate は正しい。
|
||||
let prefix = prefix_bytes(history);
|
||||
let last = records.last().expect("records non-empty");
|
||||
let ref_bytes = prefix[last.history_len.min(history.len())];
|
||||
if ref_bytes == 0 || last.input_total_tokens == 0 {
|
||||
return TokenEstimate {
|
||||
tokens: 0,
|
||||
source: EstimateSource::Extrapolated,
|
||||
};
|
||||
}
|
||||
let tokens =
|
||||
(removed_bytes as u128 * last.input_total_tokens as u128 / ref_bytes as u128) as u64;
|
||||
let source = if last.history_len == history.len() {
|
||||
EstimateSource::Measured
|
||||
} else {
|
||||
EstimateSource::Extrapolated
|
||||
};
|
||||
TokenEstimate { tokens, source }
|
||||
}
|
||||
|
||||
// ── Worker に生やす公開 API ───────────────────────────────────────────────
|
||||
|
||||
impl<C: LlmClient, St: Store> Worker<C, St> {
|
||||
/// 現在の history 全体の推定トークン数。
|
||||
///
|
||||
/// 最後の measurement と、その後に追加された未測定分の byte/4 外挿。
|
||||
pub fn total_tokens(&self) -> TokenEstimate {
|
||||
let usage = self.usage_history();
|
||||
llm_engine::token_counter::total_tokens(self.history(), &usage)
|
||||
}
|
||||
|
||||
/// 任意の history index 時点でのプロンプト全長推定。
|
||||
///
|
||||
/// `total_tokens()` と同じ accounting を任意位置で評価する版。
|
||||
/// memory extract trigger が
|
||||
/// `total_tokens_at(now) - total_tokens_at(pointer)` で
|
||||
/// pointer 以降に増えたプロンプト長を測るのに使う。
|
||||
pub fn total_tokens_at(&self, history_len: usize) -> TokenEstimate {
|
||||
let usage = self.usage_history();
|
||||
llm_engine::token_counter::total_tokens_at(self.history(), &usage, history_len)
|
||||
}
|
||||
|
||||
/// 末尾から `retained` トークン以上を残すための分割位置。
|
||||
///
|
||||
/// `history[..cut.index]` が要約/破棄される側、`history[cut.index..]` が残る側。
|
||||
pub fn split_for_retained(&self, retained: u64) -> SplitPoint {
|
||||
let usage = self.usage_history();
|
||||
split_for_retained_impl(self.history(), &usage, retained)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn msg(text: &str) -> Item {
|
||||
Item::user_message(text)
|
||||
}
|
||||
|
||||
fn record(history_len: usize, tokens: u64) -> UsageRecord {
|
||||
UsageRecord {
|
||||
history_len,
|
||||
input_total_tokens: tokens,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
output_tokens: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_returns_zero_when_current_below_retained() {
|
||||
let history = vec![msg("a"), msg("b")];
|
||||
let records = vec![record(2, 50)];
|
||||
let cut = split_for_retained_impl(&history, &records, 1000);
|
||||
assert_eq!(cut.index, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_uses_current_occupancy_as_raw_byte_rate() {
|
||||
// Compact retained split does not treat the intermediate record at
|
||||
// len=2 as a raw prefix boundary. It uses the current occupancy
|
||||
// estimate (len=4 → 300) as a serialized-byte rate and keeps the
|
||||
// smallest item-granular suffix whose raw size covers retained=200.
|
||||
let history = vec![msg("a"), msg("b"), msg("c"), msg("d")];
|
||||
let records = vec![record(2, 100), record(4, 300)];
|
||||
let cut = split_for_retained_impl(&history, &records, 200);
|
||||
assert_eq!(cut.index, 1);
|
||||
assert_eq!(cut.source, EstimateSource::Measured);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_does_not_use_non_current_measurements_as_cut_boundaries() {
|
||||
let history = vec![msg("aaaaaa"), msg("bbbbbb"), msg("cccccc"), msg("dddddd")];
|
||||
let records = vec![record(1, 50), record(4, 400)];
|
||||
let cut = split_for_retained_impl(&history, &records, 250);
|
||||
assert_eq!(cut.index, 1);
|
||||
assert_eq!(cut.source, EstimateSource::Measured);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_ignores_non_monotonic_usage_spike_for_retained_tail() {
|
||||
let history: Vec<Item> = (0..20)
|
||||
.map(|idx| msg(&format!("message-{idx}-{}", "x".repeat(100))))
|
||||
.collect();
|
||||
let records = vec![
|
||||
record(2, 900), // request-time spike after pruning/projection
|
||||
record(20, 1000),
|
||||
];
|
||||
let cut = split_for_retained_impl(&history, &records, 100);
|
||||
|
||||
// The old prefix-crossing logic picked index 2 because 900 >=
|
||||
// 1000-100, retaining almost the whole persisted history. The compact
|
||||
// split must instead use raw suffix size and keep only the tail needed
|
||||
// for the retained budget.
|
||||
assert!(cut.index > 10, "cut.index = {}", cut.index);
|
||||
assert_eq!(cut.source, EstimateSource::Measured);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_all_when_retained_zero() {
|
||||
let history = vec![msg("a"), msg("b")];
|
||||
let records = vec![record(2, 100)];
|
||||
let cut = split_for_retained_impl(&history, &records, 0);
|
||||
assert_eq!(cut.index, 2);
|
||||
}
|
||||
|
||||
fn tool_result_with(summary: &str, content: Option<&str>) -> Item {
|
||||
match content {
|
||||
Some(c) => Item::tool_result_with_content("call", summary, c),
|
||||
None => Item::tool_result("call", summary),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_estimates_for_prune_returns_every_prefix_boundary() {
|
||||
let history = vec![msg("a"), msg("b"), msg("c")];
|
||||
let estimates = token_estimates_for_prune_impl(&history, &[record(3, 300)]);
|
||||
assert_eq!(estimates.len(), history.len() + 1);
|
||||
assert_eq!(estimates[0].tokens, 0);
|
||||
assert_eq!(estimates[3].tokens, 300);
|
||||
assert_eq!(estimates[3].source, EstimateSource::Measured);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_estimates_for_prune_propagates_no_data() {
|
||||
let history = vec![msg("a"), msg("b")];
|
||||
let estimates = token_estimates_for_prune_impl(&history, &[]);
|
||||
assert_eq!(estimates.len(), history.len() + 1);
|
||||
assert_eq!(estimates[0].source, EstimateSource::Measured);
|
||||
assert_eq!(estimates[1].source, EstimateSource::NoData);
|
||||
assert_eq!(estimates[2].source, EstimateSource::NoData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn savings_for_prune_skips_non_toolresult_indices() {
|
||||
let history = vec![msg("a"), msg("b"), msg("c")];
|
||||
// indices point at plain messages, not ToolResult → 0 savings.
|
||||
let est = savings_for_prune_impl(&history, &[record(3, 300)], &[0, 1, 2]);
|
||||
assert_eq!(est.tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn savings_for_prune_skips_content_none_items() {
|
||||
let history = vec![
|
||||
msg("user"),
|
||||
tool_result_with("s1", None),
|
||||
tool_result_with("s2", None),
|
||||
];
|
||||
let est = savings_for_prune_impl(&history, &[record(3, 300)], &[1, 2]);
|
||||
assert_eq!(est.tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn savings_for_prune_counts_only_content_delta() {
|
||||
// 1 item with big content vs the same structure without content.
|
||||
let big = "x".repeat(400);
|
||||
let history = vec![
|
||||
msg("user"),
|
||||
tool_result_with("summary", Some(&big)),
|
||||
msg("tail"),
|
||||
];
|
||||
// 1 record at end so rate = tokens / total_bytes
|
||||
let total_bytes: u64 = history.iter().map(item_bytes).sum();
|
||||
let records = vec![record(history.len(), total_bytes)]; // rate = 1 tok/byte
|
||||
let est = savings_for_prune_impl(&history, &records, &[1]);
|
||||
// saved bytes ≈ size of the big content payload; with rate=1 it
|
||||
// should be close to 400 and far from the full item bytes.
|
||||
let full_item_bytes = item_bytes(&history[1]);
|
||||
assert!(est.tokens > 0);
|
||||
assert!(est.tokens < full_item_bytes);
|
||||
assert!(est.tokens >= 400);
|
||||
assert_eq!(est.source, EstimateSource::Measured);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn savings_for_prune_no_records_falls_back_to_bytes() {
|
||||
let history = vec![msg("u"), tool_result_with("s", Some("hello world"))];
|
||||
let est = savings_for_prune_impl(&history, &[], &[1]);
|
||||
assert_eq!(est.source, EstimateSource::NoData);
|
||||
assert!(est.tokens > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn savings_for_prune_extrapolated_when_history_grew_past_measurement() {
|
||||
let big = "x".repeat(200);
|
||||
let history = vec![
|
||||
msg("u1"),
|
||||
tool_result_with("s", Some(&big)),
|
||||
msg("u2"), // added after the last measurement
|
||||
];
|
||||
let records = vec![record(2, 100)];
|
||||
let est = savings_for_prune_impl(&history, &records, &[1]);
|
||||
assert_eq!(est.source, EstimateSource::Extrapolated);
|
||||
assert!(est.tokens > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn savings_for_prune_empty_indices_is_zero() {
|
||||
let history = vec![msg("a")];
|
||||
let est = savings_for_prune_impl(&history, &[record(1, 100)], &[]);
|
||||
assert_eq!(est.tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn savings_for_prune_ignores_out_of_range_indices() {
|
||||
let history = vec![msg("a")];
|
||||
let est = savings_for_prune_impl(&history, &[record(1, 100)], &[99]);
|
||||
assert_eq!(est.tokens, 0);
|
||||
}
|
||||
|
||||
fn tc(call_id: &str) -> Item {
|
||||
Item::tool_call(call_id, "Read", "{}")
|
||||
}
|
||||
|
||||
fn tr(call_id: &str) -> Item {
|
||||
Item::tool_result(call_id, "summary")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balance_noop_on_clean_message_boundary() {
|
||||
let history = vec![msg("a"), msg("b"), msg("c")];
|
||||
assert_eq!(balance_to_pair_boundary(&history, 2), 2);
|
||||
assert_eq!(balance_to_pair_boundary(&history, 0), 0);
|
||||
assert_eq!(balance_to_pair_boundary(&history, 3), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balance_retreats_from_inside_parallel_tool_results() {
|
||||
// [Msg, TC_a, TC_b, TC_c, TR_a, TR_b, TR_c]
|
||||
// cut=5 → retained=[TR_b, TR_c]。TR_c の TC は idx=3、TR_b は idx=2 →
|
||||
// idx=2 まで後退。だが retained に TR_a (idx=4) が新たに入り、その TC_a
|
||||
// は idx=1 でまだ外 → 連鎖後退で最終的に idx=1。retained は
|
||||
// [TC_a, TC_b, TC_c, TR_a, TR_b, TR_c]。
|
||||
let history = vec![
|
||||
msg("u"),
|
||||
tc("a"),
|
||||
tc("b"),
|
||||
tc("c"),
|
||||
tr("a"),
|
||||
tr("b"),
|
||||
tr("c"),
|
||||
];
|
||||
assert_eq!(balance_to_pair_boundary(&history, 5), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balance_retreats_between_call_and_result() {
|
||||
// [TC_a, TR_a, TC_b, TR_b]。cut=3 → retained=[TR_b] orphan。
|
||||
// TC_b は idx=2 → cut=2。retained=[TC_b, TR_b]。
|
||||
let history = vec![tc("a"), tr("a"), tc("b"), tr("b")];
|
||||
assert_eq!(balance_to_pair_boundary(&history, 3), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balance_cascades_through_nested_pairs() {
|
||||
// [TC_a, TC_b, TR_b, TR_a, TC_c, TR_c]。cut=3 → retained=[TR_a, TC_c, TR_c]。
|
||||
// TR_a の TC は idx=0 → cut=0。retained=full。
|
||||
let history = vec![tc("a"), tc("b"), tr("b"), tr("a"), tc("c"), tr("c")];
|
||||
assert_eq!(balance_to_pair_boundary(&history, 3), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balance_noop_when_cut_at_pair_boundary() {
|
||||
// [TC_a, TR_a, Msg, TC_b, TR_b]。cut=2 → retained=[Msg, TC_b, TR_b] balanced。
|
||||
let history = vec![tc("a"), tr("a"), msg("u"), tc("b"), tr("b")];
|
||||
assert_eq!(balance_to_pair_boundary(&history, 2), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balance_handles_orphan_result_without_matching_call() {
|
||||
// ToolCall がそもそも存在しない ToolResult は触らない(壊れた history は
|
||||
// ここでは直しようがない)。cut=1 → そのまま 1 を返す。
|
||||
let history = vec![msg("u"), tr("zombie")];
|
||||
assert_eq!(balance_to_pair_boundary(&history, 1), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balance_keeps_cut_when_call_is_inside_retained() {
|
||||
// [Msg, TC_a, TR_a]。cut=1 → retained=[TC_a, TR_a]。TR_a の call_pos=1 >= idx=1。OK。
|
||||
let history = vec![msg("u"), tc("a"), tr("a")];
|
||||
assert_eq!(balance_to_pair_boundary(&history, 1), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_for_retained_aligns_to_pair_boundary() {
|
||||
// 並列 TC*3 / TR*3 ターン後に Msg を 1 件足し、retained=Msg のサイズ相当に
|
||||
// 設定。トークン的には cut=末尾近くだが、orphan を避けるため TC 群の手前
|
||||
// まで後退するはず。
|
||||
let history = vec![
|
||||
msg("user"),
|
||||
tc("a"),
|
||||
tc("b"),
|
||||
tc("c"),
|
||||
tr("a"),
|
||||
tr("b"),
|
||||
tr("c"),
|
||||
msg("tail"),
|
||||
];
|
||||
let total_bytes: u64 = history.iter().map(item_bytes).sum();
|
||||
let records = vec![record(history.len(), total_bytes)]; // rate = 1 tok/byte
|
||||
// tail の item_bytes 相当のみ retain したい。
|
||||
let tail_tokens = item_bytes(&history[7]);
|
||||
let cut = split_for_retained_impl(&history, &records, tail_tokens);
|
||||
// token 単独だと cut は 7(tail のみ retained)になるが、retained 先頭が
|
||||
// Msg なら balance しなくて OK。balance helper の no-op を確認する意味も込めて
|
||||
// index == 7 を期待する。
|
||||
assert_eq!(cut.index, 7);
|
||||
|
||||
// 逆に retained をやや増やしてトークン的に cut=6(TR_c のみ retained)に
|
||||
// させると、TR_c は orphan なので balance が 1 まで後退するはず。
|
||||
let big_retain = tail_tokens + item_bytes(&history[6]);
|
||||
let cut = split_for_retained_impl(&history, &records, big_retain);
|
||||
assert_eq!(cut.index, 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
//! Tracks per-LLM-request Usage measurements within a Worker run.
|
||||
//!
|
||||
//! Bridge between two sync touchpoints in the Engine lifecycle:
|
||||
//!
|
||||
//! - **`pre_llm_request` hook** (async, but synchronously accessed via the
|
||||
//! tracker): captures `history.len()` at the moment a request goes out.
|
||||
//! - **`on_usage` callback** (sync closure): receives the aggregated final
|
||||
//! `UsageEvent` for that request after the stream completes.
|
||||
//!
|
||||
//! Pairing the two yields one `UsageRecord` per LLM call. Worker drains them
|
||||
//! in `persist_turn` and writes them as `LogEntry::LlmUsage` entries.
|
||||
//!
|
||||
//! Multiple LLM calls per Worker run (tool loop) are supported: each call
|
||||
//! produces its own `(history_len, UsageEvent)` pair, and the records are
|
||||
//! buffered in chronological order.
|
||||
|
||||
use std::sync::Mutex;
|
||||
|
||||
use llm_engine::UsageRecord;
|
||||
use llm_engine::timeline::event::UsageEvent;
|
||||
|
||||
/// One drained measurement: the underlying `UsageRecord` plus an optional
|
||||
/// `correlation_id` stamped by the prune projection (or any other future
|
||||
/// upstream observer) so that downstream metrics emitted alongside this
|
||||
/// record can be joined to it after the fact.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct RecordedUsage {
|
||||
pub(crate) record: UsageRecord,
|
||||
pub(crate) correlation_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Shared between the pre-request hook, the `on_usage` callback, and Worker.
|
||||
pub(crate) struct UsageTracker {
|
||||
/// `history.len()` captured at the most recent `pre_llm_request`.
|
||||
/// Cleared when paired with an incoming `on_usage` event.
|
||||
pending_history_len: Mutex<Option<usize>>,
|
||||
/// Optional `correlation_id` set by an upstream observer (currently
|
||||
/// the prune projection on `Fired`). Paired into the next
|
||||
/// `RecordedUsage` and cleared. Skips that don't fire leave this
|
||||
/// `None`, so the resulting record carries no correlation.
|
||||
pending_correlation_id: Mutex<Option<String>>,
|
||||
/// Records accumulated during the current run; drained by Worker.
|
||||
pending_records: Mutex<Vec<RecordedUsage>>,
|
||||
}
|
||||
|
||||
impl UsageTracker {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
pending_history_len: Mutex::new(None),
|
||||
pending_correlation_id: Mutex::new(None),
|
||||
pending_records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Called from a `pre_llm_request` hook with the current history length.
|
||||
pub(crate) fn note_request(&self, history_len: usize) {
|
||||
*self.pending_history_len.lock().unwrap() = Some(history_len);
|
||||
}
|
||||
|
||||
/// Stash a `correlation_id` to be paired into the next `RecordedUsage`.
|
||||
/// Currently invoked by the prune observer on `Fired` so that the
|
||||
/// `prune.fire` metric and the `prune.post_request` metric (emitted
|
||||
/// alongside the resulting `LlmUsage`) carry the same join key.
|
||||
///
|
||||
/// Overwrites any previous unconsumed value — by construction the
|
||||
/// observer fires at most once per outgoing LLM request, immediately
|
||||
/// before the pre-request hook captures `history_len`.
|
||||
pub(crate) fn note_correlation_id(&self, id: String) {
|
||||
*self.pending_correlation_id.lock().unwrap() = Some(id);
|
||||
}
|
||||
|
||||
/// Called from the `on_usage` callback with the aggregated final
|
||||
/// UsageEvent. If a `history_len` was previously stashed via
|
||||
/// `note_request`, builds a `RecordedUsage` and pushes it onto the
|
||||
/// buffer. If not (e.g. test code that fires Usage outside a request),
|
||||
/// drops the event.
|
||||
pub(crate) fn record_usage(&self, event: &UsageEvent) {
|
||||
let history_len = match self.pending_history_len.lock().unwrap().take() {
|
||||
Some(n) => n,
|
||||
None => return,
|
||||
};
|
||||
let correlation_id = self.pending_correlation_id.lock().unwrap().take();
|
||||
// UsageEvent.input_tokens は scheme 層で「占有量(プロンプト全長)」に
|
||||
// 正規化済みである前提(Anthropic は cache_read + cache_creation を
|
||||
// 加算して emit する)。
|
||||
let input_total = event.input_tokens.unwrap_or(0);
|
||||
let cache_read = event.cache_read_input_tokens.unwrap_or(0);
|
||||
let cache_write = event.cache_creation_input_tokens.unwrap_or(0);
|
||||
let output = event.output_tokens.unwrap_or(0);
|
||||
self.pending_records.lock().unwrap().push(RecordedUsage {
|
||||
record: UsageRecord {
|
||||
history_len,
|
||||
input_total_tokens: input_total,
|
||||
cache_read_tokens: cache_read,
|
||||
cache_write_tokens: cache_write,
|
||||
output_tokens: output,
|
||||
},
|
||||
correlation_id,
|
||||
});
|
||||
}
|
||||
|
||||
/// Return a clone of the accumulated `UsageRecord`s without clearing them.
|
||||
/// Used by request-time circuit breakers that need the same occupancy
|
||||
/// projection as Worker persistence while the run is still active.
|
||||
pub(crate) fn records(&self) -> Vec<UsageRecord> {
|
||||
self.pending_records
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|r| r.record.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Drain accumulated records. Called by Worker after a run completes,
|
||||
/// before persisting the turn.
|
||||
pub(crate) fn drain(&self) -> Vec<RecordedUsage> {
|
||||
std::mem::take(&mut *self.pending_records.lock().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_event(input: u64, cache_read: u64, cache_write: u64, output: u64) -> UsageEvent {
|
||||
UsageEvent {
|
||||
input_tokens: Some(input),
|
||||
output_tokens: Some(output),
|
||||
total_tokens: Some(input + output),
|
||||
cache_read_input_tokens: Some(cache_read),
|
||||
cache_creation_input_tokens: Some(cache_write),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pairs_history_len_with_usage_event() {
|
||||
let tracker = UsageTracker::new();
|
||||
tracker.note_request(5);
|
||||
tracker.record_usage(&make_event(1000, 800, 100, 42));
|
||||
|
||||
let records = tracker.drain();
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].record.history_len, 5);
|
||||
assert_eq!(records[0].record.input_total_tokens, 1000);
|
||||
assert_eq!(records[0].record.cache_read_tokens, 800);
|
||||
assert_eq!(records[0].record.cache_write_tokens, 100);
|
||||
assert_eq!(records[0].record.output_tokens, 42);
|
||||
assert!(records[0].correlation_id.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_clones_without_clearing() {
|
||||
let tracker = UsageTracker::new();
|
||||
tracker.note_request(1);
|
||||
tracker.record_usage(&make_event(10, 0, 0, 5));
|
||||
|
||||
let records = tracker.records();
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].history_len, 1);
|
||||
assert_eq!(records[0].input_total_tokens, 10);
|
||||
assert_eq!(tracker.records().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_clears_buffer() {
|
||||
let tracker = UsageTracker::new();
|
||||
tracker.note_request(1);
|
||||
tracker.record_usage(&make_event(10, 0, 0, 5));
|
||||
assert_eq!(tracker.drain().len(), 1);
|
||||
assert_eq!(tracker.drain().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_without_pending_history_len_is_dropped() {
|
||||
let tracker = UsageTracker::new();
|
||||
tracker.record_usage(&make_event(10, 0, 0, 5));
|
||||
assert_eq!(tracker.drain().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_requests_in_one_run() {
|
||||
let tracker = UsageTracker::new();
|
||||
tracker.note_request(5);
|
||||
tracker.record_usage(&make_event(100, 0, 0, 20));
|
||||
tracker.note_request(10);
|
||||
tracker.record_usage(&make_event(200, 50, 0, 30));
|
||||
|
||||
let records = tracker.drain();
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0].record.history_len, 5);
|
||||
assert_eq!(records[1].record.history_len, 10);
|
||||
assert_eq!(records[1].record.cache_read_tokens, 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correlation_id_pairs_with_next_record_only() {
|
||||
let tracker = UsageTracker::new();
|
||||
// Stash an ID, then run a request → the ID should land on this record.
|
||||
tracker.note_correlation_id("abc".into());
|
||||
tracker.note_request(5);
|
||||
tracker.record_usage(&make_event(100, 0, 0, 20));
|
||||
// Next request without a fresh stash → no correlation_id.
|
||||
tracker.note_request(10);
|
||||
tracker.record_usage(&make_event(200, 50, 0, 30));
|
||||
|
||||
let records = tracker.drain();
|
||||
assert_eq!(records.len(), 2);
|
||||
assert_eq!(records[0].correlation_id.as_deref(), Some("abc"));
|
||||
assert!(records[1].correlation_id.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,878 @@
|
||||
//! Compact worker state and the four tools that drive it.
|
||||
//!
|
||||
//! The compact worker is a disposable `Engine` instance spun up by
|
||||
//! [`Worker::compact`]. It receives the history to summarise plus a list of
|
||||
//! default reference files (from the session-lifetime `Tracker`) and runs
|
||||
//! a tool-driven LLM loop. The tools here let it:
|
||||
//!
|
||||
//! - `read_file` — inspect referenced files (reuses `tools::read_tool`)
|
||||
//! - `mark_read_required(path, offset?, limit?)` — nominate a file whose
|
||||
//! contents should be injected into the compacted context as an
|
||||
//! auto-read system message
|
||||
//! - `add_reference(path)` — nominate a file the next session should
|
||||
//! know about by name only (contents not included)
|
||||
//! - `write_summary(text)` — deliver (or overwrite) the structured summary
|
||||
//!
|
||||
//! Everything the worker decides ends up in [`CompactWorkerContext`],
|
||||
//! which `Worker::compact` drains after the loop and turns into the
|
||||
//! compacted session's opening system messages.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use llm_engine::Item;
|
||||
use llm_engine::interceptor::{Interceptor, PreRequestAction, PreToolAction, ToolCallInfo};
|
||||
use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput, ToolResult};
|
||||
use serde::Deserialize;
|
||||
use tools::ScopedFs;
|
||||
|
||||
use crate::compact::usage_tracker::UsageTracker;
|
||||
use crate::fs_view::{ReadRequirement, slice_lines};
|
||||
|
||||
/// Aggregated output of a compact worker run.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(crate) struct CompactWorkerContext {
|
||||
pub read_required: Vec<ReadRequirement>,
|
||||
pub references: Vec<PathBuf>,
|
||||
pub summary: Option<String>,
|
||||
/// Tokens already consumed by `mark_read_required` calls.
|
||||
pub auto_read_consumed: u64,
|
||||
/// Aggregate cap. `0` treats the budget as disabled.
|
||||
pub auto_read_budget: u64,
|
||||
}
|
||||
|
||||
impl CompactWorkerContext {
|
||||
pub(crate) fn with_budget(auto_read_budget: u64) -> Self {
|
||||
Self {
|
||||
auto_read_budget,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn remaining_budget(&self) -> u64 {
|
||||
self.auto_read_budget
|
||||
.saturating_sub(self.auto_read_consumed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Input to `mark_read_required`.
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
struct MarkParams {
|
||||
/// Absolute path to the file.
|
||||
pub file_path: PathBuf,
|
||||
/// 0-based line offset.
|
||||
#[serde(default)]
|
||||
pub offset: Option<usize>,
|
||||
/// Maximum number of lines to inject.
|
||||
#[serde(default)]
|
||||
pub limit: Option<usize>,
|
||||
}
|
||||
|
||||
/// Input to `add_reference`.
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
struct ReferenceParams {
|
||||
/// Absolute path to the file.
|
||||
pub file_path: PathBuf,
|
||||
}
|
||||
|
||||
/// Input to `write_summary`.
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
struct SummaryParams {
|
||||
/// Full structured summary text (overwrites any previous call).
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// Input to `search_session_log`.
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
struct SearchSessionParams {
|
||||
/// Case-insensitive substring to search in compact-target history.
|
||||
pub query: String,
|
||||
/// 0-based item offset to start searching from.
|
||||
#[serde(default)]
|
||||
pub offset: Option<usize>,
|
||||
/// Maximum number of hits to return.
|
||||
#[serde(default)]
|
||||
pub limit: Option<usize>,
|
||||
}
|
||||
|
||||
/// Input to `read_session_items`.
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
struct ReadSessionParams {
|
||||
/// 0-based compact-target history item offset.
|
||||
pub offset: usize,
|
||||
/// Maximum number of items to return.
|
||||
pub limit: usize,
|
||||
/// `compact` omits tool arguments/full results; `full` includes message text and tool result content.
|
||||
#[serde(default = "default_session_read_mode")]
|
||||
pub mode: String,
|
||||
}
|
||||
|
||||
fn default_session_read_mode() -> String {
|
||||
"compact".to_string()
|
||||
}
|
||||
|
||||
const SESSION_TOOL_MAX_OUTPUT_TOKENS: u64 = 12_000;
|
||||
const SESSION_SEARCH_MAX_RESULTS: usize = 50;
|
||||
const SESSION_READ_MAX_ITEMS: usize = 80;
|
||||
|
||||
const MARK_DESCRIPTION: &str = "Inject a file's contents into the compacted context so the \
|
||||
next session starts with it already read. Use this for files the next task needs in full. \
|
||||
Optionally specify `offset` (0-based line) and `limit` (line count) to inject only a slice. \
|
||||
Counts against `auto_read_budget`; overflow returns an error and the mark is not recorded. \
|
||||
Paths must be absolute.";
|
||||
|
||||
const REFERENCE_DESCRIPTION: &str = "Record a file path as a named reference in the compacted \
|
||||
context without injecting its contents. Use for files that are contextually relevant but \
|
||||
whose current content the next session can fetch on demand.";
|
||||
|
||||
const SUMMARY_DESCRIPTION: &str = "Provide the final structured summary text. Subsequent calls \
|
||||
replace the previous content; only the last call is used. Must be called before the compact run \
|
||||
ends or compaction fails.";
|
||||
|
||||
const SEARCH_SESSION_DESCRIPTION: &str = "Search the compact-target session history by \
|
||||
case-insensitive substring. Returns item indexes and compact snippets. Use this when the initial \
|
||||
overview is not enough to identify which part of the session matters. Results are bounded; narrow \
|
||||
the query if important details are omitted.";
|
||||
|
||||
const READ_SESSION_DESCRIPTION: &str = "Read a bounded range of compact-target session history \
|
||||
items by 0-based index. mode='compact' omits tool arguments, full tool results, and reasoning \
|
||||
bodies; mode='full' includes message text and tool result content but still remains bounded. Use \
|
||||
this to verify details before writing the summary.";
|
||||
|
||||
struct SessionLogToolState {
|
||||
items: Arc<Vec<Item>>,
|
||||
}
|
||||
|
||||
struct SearchSessionLogTool {
|
||||
state: Arc<SessionLogToolState>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for SearchSessionLogTool {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: SearchSessionParams = serde_json::from_str(input_json).map_err(|e| {
|
||||
ToolError::InvalidArgument(format!("invalid search_session_log input: {e}"))
|
||||
})?;
|
||||
let query = params.query.trim().to_lowercase();
|
||||
if query.is_empty() {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"search_session_log query must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
let offset = params.offset.unwrap_or(0).min(self.state.items.len());
|
||||
let limit = params
|
||||
.limit
|
||||
.unwrap_or(20)
|
||||
.clamp(1, SESSION_SEARCH_MAX_RESULTS);
|
||||
let mut hits = Vec::new();
|
||||
for (idx, item) in self.state.items.iter().enumerate().skip(offset) {
|
||||
let haystack = session_item_search_text(item).to_lowercase();
|
||||
if haystack.contains(&query) {
|
||||
hits.push(format_session_item(
|
||||
idx,
|
||||
item,
|
||||
SessionReadMode::Compact,
|
||||
600,
|
||||
));
|
||||
if hits.len() >= limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut content = hits.join("\n\n");
|
||||
let truncated = truncate_to_token_budget(&mut content, SESSION_TOOL_MAX_OUTPUT_TOKENS);
|
||||
let summary = if hits.is_empty() {
|
||||
format!("No session log hits for {query:?} from item offset {offset}.")
|
||||
} else if truncated {
|
||||
format!(
|
||||
"Found {} session log hit(s) for {query:?}; output truncated. Narrow the query.",
|
||||
hits.len()
|
||||
)
|
||||
} else {
|
||||
format!("Found {} session log hit(s) for {query:?}.", hits.len())
|
||||
};
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: (!content.is_empty()).then_some(content),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct ReadSessionItemsTool {
|
||||
state: Arc<SessionLogToolState>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for ReadSessionItemsTool {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: ReadSessionParams = serde_json::from_str(input_json).map_err(|e| {
|
||||
ToolError::InvalidArgument(format!("invalid read_session_items input: {e}"))
|
||||
})?;
|
||||
let mode = SessionReadMode::parse(¶ms.mode)?;
|
||||
let offset = params.offset.min(self.state.items.len());
|
||||
let limit = params.limit.clamp(1, SESSION_READ_MAX_ITEMS);
|
||||
let end = offset.saturating_add(limit).min(self.state.items.len());
|
||||
let mut blocks = Vec::new();
|
||||
for idx in offset..end {
|
||||
blocks.push(format_session_item(
|
||||
idx,
|
||||
&self.state.items[idx],
|
||||
mode,
|
||||
4_000,
|
||||
));
|
||||
}
|
||||
let mut content = blocks.join("\n\n");
|
||||
let truncated = truncate_to_token_budget(&mut content, SESSION_TOOL_MAX_OUTPUT_TOKENS);
|
||||
let summary = if truncated {
|
||||
format!(
|
||||
"Read session items {offset}..{end} in {mode:?} mode; output truncated. Narrow the range."
|
||||
)
|
||||
} else {
|
||||
format!("Read session items {offset}..{end} in {mode:?} mode.")
|
||||
};
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: (!content.is_empty()).then_some(content),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum SessionReadMode {
|
||||
Compact,
|
||||
Full,
|
||||
}
|
||||
|
||||
impl SessionReadMode {
|
||||
fn parse(value: &str) -> Result<Self, ToolError> {
|
||||
match value {
|
||||
"compact" => Ok(Self::Compact),
|
||||
"full" => Ok(Self::Full),
|
||||
other => Err(ToolError::InvalidArgument(format!(
|
||||
"invalid read_session_items mode {other:?}; expected 'compact' or 'full'"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn session_item_search_text(item: &Item) -> String {
|
||||
match item {
|
||||
Item::Message { role, content, .. } => format!(
|
||||
"{:?} {}",
|
||||
role,
|
||||
content
|
||||
.iter()
|
||||
.map(|p| p.as_text())
|
||||
.collect::<Vec<_>>()
|
||||
.join("")
|
||||
),
|
||||
Item::ToolCall {
|
||||
name, arguments, ..
|
||||
} => format!("tool_call {name} {arguments}"),
|
||||
Item::ToolResult {
|
||||
summary, content, ..
|
||||
} => format!(
|
||||
"tool_result {summary} {}",
|
||||
content.as_deref().unwrap_or_default()
|
||||
),
|
||||
Item::Reasoning { text, summary, .. } => format!("reasoning {text} {}", summary.join(" ")),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_session_item(idx: usize, item: &Item, mode: SessionReadMode, max_chars: usize) -> String {
|
||||
match item {
|
||||
Item::Message { role, content, .. } => {
|
||||
let text = content
|
||||
.iter()
|
||||
.map(|p| p.as_text())
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
format!(
|
||||
"[{idx} Message {:?}] {}",
|
||||
role,
|
||||
truncate_chars(&text, max_chars)
|
||||
)
|
||||
}
|
||||
Item::ToolCall {
|
||||
name, arguments, ..
|
||||
} => match mode {
|
||||
SessionReadMode::Compact => format!("[{idx} ToolCall] {name} (arguments omitted)"),
|
||||
SessionReadMode::Full => format!(
|
||||
"[{idx} ToolCall] {name}\narguments: {}",
|
||||
truncate_chars(arguments, max_chars)
|
||||
),
|
||||
},
|
||||
Item::ToolResult {
|
||||
summary,
|
||||
content,
|
||||
is_error,
|
||||
..
|
||||
} => match mode {
|
||||
SessionReadMode::Compact => format!(
|
||||
"[{idx} ToolResult{}] {} (content omitted)",
|
||||
if *is_error { " error" } else { "" },
|
||||
truncate_chars(summary, 800)
|
||||
),
|
||||
SessionReadMode::Full => format!(
|
||||
"[{idx} ToolResult{}] {}\ncontent: {}",
|
||||
if *is_error { " error" } else { "" },
|
||||
truncate_chars(summary, 800),
|
||||
truncate_chars(content.as_deref().unwrap_or(""), max_chars)
|
||||
),
|
||||
},
|
||||
Item::Reasoning { summary, .. } => match mode {
|
||||
SessionReadMode::Compact => format!(
|
||||
"[{idx} Reasoning] {} (body omitted)",
|
||||
truncate_chars(&summary.join(" "), 800)
|
||||
),
|
||||
SessionReadMode::Full => format!(
|
||||
"[{idx} Reasoning] {} (body omitted)",
|
||||
truncate_chars(&summary.join(" "), 800)
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_chars(text: &str, max_chars: usize) -> String {
|
||||
if text.chars().count() <= max_chars {
|
||||
return text.to_string();
|
||||
}
|
||||
let mut out = text.chars().take(max_chars).collect::<String>();
|
||||
out.push_str("… [truncated]");
|
||||
out
|
||||
}
|
||||
|
||||
fn truncate_to_token_budget(text: &mut String, max_tokens: u64) -> bool {
|
||||
let max_bytes = max_tokens.saturating_mul(4) as usize;
|
||||
if text.len() <= max_bytes {
|
||||
return false;
|
||||
}
|
||||
let mut cut = 0;
|
||||
for (idx, _) in text.char_indices() {
|
||||
if idx > max_bytes {
|
||||
break;
|
||||
}
|
||||
cut = idx;
|
||||
}
|
||||
text.truncate(cut);
|
||||
text.push_str("\n… [session tool output truncated]");
|
||||
true
|
||||
}
|
||||
|
||||
struct MarkReadRequiredTool {
|
||||
fs: ScopedFs,
|
||||
ctx: Arc<Mutex<CompactWorkerContext>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for MarkReadRequiredTool {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: MarkParams = serde_json::from_str(input_json).map_err(|e| {
|
||||
ToolError::InvalidArgument(format!("invalid mark_read_required input: {e}"))
|
||||
})?;
|
||||
|
||||
// Read the file through the shared ScopedFs so scope and I/O
|
||||
// errors surface the same way the regular `read_file` tool does.
|
||||
let bytes = self
|
||||
.fs
|
||||
.read_bytes(¶ms.file_path)
|
||||
.map_err(|e| ToolError::ExecutionFailed(format!("read failed: {e}")))?;
|
||||
let text = String::from_utf8_lossy(&bytes);
|
||||
let slice = slice_lines(&text, params.offset.unwrap_or(0), params.limit);
|
||||
let estimated_tokens = estimate_tokens(slice.len());
|
||||
|
||||
let mut guard = self.ctx.lock().expect("compact worker context poisoned");
|
||||
let budget = guard.auto_read_budget;
|
||||
let would_consume = guard.auto_read_consumed.saturating_add(estimated_tokens);
|
||||
if budget > 0 && would_consume > budget {
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"auto-read budget exhausted ({budget} tokens). Remove an existing mark or use \
|
||||
add_reference instead."
|
||||
)));
|
||||
}
|
||||
guard.read_required.push(ReadRequirement {
|
||||
path: params.file_path.clone(),
|
||||
offset: params.offset,
|
||||
limit: params.limit,
|
||||
});
|
||||
guard.auto_read_consumed = would_consume;
|
||||
let remaining = guard.remaining_budget();
|
||||
drop(guard);
|
||||
|
||||
let mut summary = format!(
|
||||
"Marked {} for auto-read (≈{estimated_tokens} tokens). \
|
||||
Budget: {remaining}/{budget} tokens remaining.",
|
||||
params.file_path.display()
|
||||
);
|
||||
if budget > 0 && remaining * 2 <= budget {
|
||||
summary.push_str(
|
||||
"\nNote: auto-read budget is at least half consumed. \
|
||||
Consider calling write_summary and finishing up soon.",
|
||||
);
|
||||
}
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
content: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct AddReferenceTool {
|
||||
ctx: Arc<Mutex<CompactWorkerContext>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for AddReferenceTool {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: ReferenceParams = serde_json::from_str(input_json)
|
||||
.map_err(|e| ToolError::InvalidArgument(format!("invalid add_reference input: {e}")))?;
|
||||
let mut guard = self.ctx.lock().expect("compact worker context poisoned");
|
||||
if !guard
|
||||
.references
|
||||
.iter()
|
||||
.any(|p| p.as_path() == params.file_path.as_path())
|
||||
{
|
||||
guard.references.push(params.file_path.clone());
|
||||
}
|
||||
Ok(ToolOutput {
|
||||
summary: format!("Added reference {}", params.file_path.display()),
|
||||
content: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct WriteSummaryTool {
|
||||
ctx: Arc<Mutex<CompactWorkerContext>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for WriteSummaryTool {
|
||||
async fn execute(
|
||||
&self,
|
||||
input_json: &str,
|
||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: SummaryParams = serde_json::from_str(input_json)
|
||||
.map_err(|e| ToolError::InvalidArgument(format!("invalid write_summary input: {e}")))?;
|
||||
let mut guard = self.ctx.lock().expect("compact worker context poisoned");
|
||||
let overwritten = guard.summary.is_some();
|
||||
guard.summary = Some(params.text);
|
||||
drop(guard);
|
||||
let note = if overwritten {
|
||||
"Summary replaced."
|
||||
} else {
|
||||
"Summary recorded."
|
||||
};
|
||||
Ok(ToolOutput {
|
||||
summary: note.to_string(),
|
||||
content: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn mark_read_required_tool(
|
||||
fs: ScopedFs,
|
||||
ctx: Arc<Mutex<CompactWorkerContext>>,
|
||||
) -> ToolDefinition {
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(MarkParams);
|
||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||
let meta = ToolMeta::new("mark_read_required")
|
||||
.description(MARK_DESCRIPTION)
|
||||
.input_schema(schema_value);
|
||||
let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool {
|
||||
fs: fs.clone(),
|
||||
ctx: ctx.clone(),
|
||||
});
|
||||
(meta, tool)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn add_reference_tool(ctx: Arc<Mutex<CompactWorkerContext>>) -> ToolDefinition {
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(ReferenceParams);
|
||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||
let meta = ToolMeta::new("add_reference")
|
||||
.description(REFERENCE_DESCRIPTION)
|
||||
.input_schema(schema_value);
|
||||
let tool: Arc<dyn Tool> = Arc::new(AddReferenceTool { ctx: ctx.clone() });
|
||||
(meta, tool)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn write_summary_tool(ctx: Arc<Mutex<CompactWorkerContext>>) -> ToolDefinition {
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(SummaryParams);
|
||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||
let meta = ToolMeta::new("write_summary")
|
||||
.description(SUMMARY_DESCRIPTION)
|
||||
.input_schema(schema_value);
|
||||
let tool: Arc<dyn Tool> = Arc::new(WriteSummaryTool { ctx: ctx.clone() });
|
||||
(meta, tool)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn search_session_log_tool(items: Arc<Vec<Item>>) -> ToolDefinition {
|
||||
let state = Arc::new(SessionLogToolState { items });
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(SearchSessionParams);
|
||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||
let meta = ToolMeta::new("search_session_log")
|
||||
.description(SEARCH_SESSION_DESCRIPTION)
|
||||
.input_schema(schema_value);
|
||||
let tool: Arc<dyn Tool> = Arc::new(SearchSessionLogTool {
|
||||
state: state.clone(),
|
||||
});
|
||||
(meta, tool)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn read_session_items_tool(items: Arc<Vec<Item>>) -> ToolDefinition {
|
||||
let state = Arc::new(SessionLogToolState { items });
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(ReadSessionParams);
|
||||
let schema_value = serde_json::to_value(schema).unwrap_or(serde_json::json!({}));
|
||||
let meta = ToolMeta::new("read_session_items")
|
||||
.description(READ_SESSION_DESCRIPTION)
|
||||
.input_schema(schema_value);
|
||||
let tool: Arc<dyn Tool> = Arc::new(ReadSessionItemsTool {
|
||||
state: state.clone(),
|
||||
});
|
||||
(meta, tool)
|
||||
})
|
||||
}
|
||||
|
||||
/// Interceptor that monitors compact-worker context occupancy.
|
||||
///
|
||||
/// `max_input_tokens` remains the hard circuit breaker. Before that point,
|
||||
/// the interceptor can persist a system warning into worker history telling
|
||||
/// the model to stop broad exploration and call `write_summary`, and can block
|
||||
/// additional exploratory tool calls once the final reserve is reached.
|
||||
pub(crate) struct CompactWorkerInterceptor {
|
||||
pub usage_tracker: Arc<UsageTracker>,
|
||||
pub max_input_tokens: u64,
|
||||
pub finish_warning_remaining_tokens: u64,
|
||||
pub final_reserve_tokens: u64,
|
||||
pub on_warning: Option<Arc<dyn Fn(String) + Send + Sync>>,
|
||||
warning_sent: AtomicBool,
|
||||
last_remaining_tokens: AtomicU64,
|
||||
}
|
||||
|
||||
impl CompactWorkerInterceptor {
|
||||
pub(crate) fn new(
|
||||
usage_tracker: Arc<UsageTracker>,
|
||||
max_input_tokens: u64,
|
||||
finish_warning_remaining_tokens: u64,
|
||||
final_reserve_tokens: u64,
|
||||
on_warning: Option<Arc<dyn Fn(String) + Send + Sync>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
usage_tracker,
|
||||
max_input_tokens,
|
||||
finish_warning_remaining_tokens,
|
||||
final_reserve_tokens,
|
||||
on_warning,
|
||||
warning_sent: AtomicBool::new(false),
|
||||
last_remaining_tokens: AtomicU64::new(max_input_tokens),
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_emit_warning(&self, remaining: u64) -> Option<Item> {
|
||||
let warning_threshold = self.finish_warning_remaining_tokens;
|
||||
let reserve_threshold = self.final_reserve_tokens;
|
||||
let should_warn = (warning_threshold > 0 && remaining <= warning_threshold)
|
||||
|| (reserve_threshold > 0 && remaining <= reserve_threshold);
|
||||
if !should_warn || self.warning_sent.swap(true, Ordering::AcqRel) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let message = format!(
|
||||
"compact worker context budget is low ({remaining}/{} tokens remaining). \
|
||||
Stop broad exploration now, read only if absolutely necessary, then call \
|
||||
`write_summary` with the final structured summary.",
|
||||
self.max_input_tokens
|
||||
);
|
||||
if let Some(cb) = self.on_warning.as_ref() {
|
||||
cb(message.clone());
|
||||
}
|
||||
Some(Item::system_message(format!(
|
||||
"[Compact worker budget warning]\n\n{message}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Interceptor for CompactWorkerInterceptor {
|
||||
async fn pre_llm_request(&self, context: &mut Vec<Item>) -> PreRequestAction {
|
||||
let records = self.usage_tracker.records();
|
||||
let estimate = llm_engine::token_counter::total_tokens(context, &records);
|
||||
if estimate.tokens > self.max_input_tokens {
|
||||
return PreRequestAction::Cancel(format!(
|
||||
"compact worker input occupancy exceeded {} tokens",
|
||||
self.max_input_tokens
|
||||
));
|
||||
}
|
||||
|
||||
let remaining = self.max_input_tokens.saturating_sub(estimate.tokens);
|
||||
self.last_remaining_tokens
|
||||
.store(remaining, Ordering::Release);
|
||||
if let Some(item) = self.maybe_emit_warning(remaining) {
|
||||
self.usage_tracker.note_request(context.len() + 1);
|
||||
return PreRequestAction::ContinueWith(vec![item]);
|
||||
}
|
||||
|
||||
self.usage_tracker.note_request(context.len());
|
||||
PreRequestAction::Continue
|
||||
}
|
||||
|
||||
async fn pre_tool_call(&self, info: &mut ToolCallInfo) -> PreToolAction {
|
||||
if self.final_reserve_tokens == 0 || info.call.name == "write_summary" {
|
||||
return PreToolAction::Continue;
|
||||
}
|
||||
let remaining = self.last_remaining_tokens.load(Ordering::Acquire);
|
||||
if remaining > self.final_reserve_tokens {
|
||||
return PreToolAction::Continue;
|
||||
}
|
||||
PreToolAction::SyntheticResult(ToolResult::error(
|
||||
info.call.id.clone(),
|
||||
"compact worker final reserve reached; do not perform more exploratory tool reads. Call `write_summary` now.",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Crude bytes→tokens estimate; good enough for budget accounting.
|
||||
fn estimate_tokens(bytes: usize) -> u64 {
|
||||
(bytes as u64).div_ceil(4)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use manifest::Scope;
|
||||
|
||||
fn make_fs(tmp: &std::path::Path) -> ScopedFs {
|
||||
let scope = Scope::writable(tmp.to_path_buf()).unwrap();
|
||||
ScopedFs::new(scope, tmp.to_path_buf())
|
||||
}
|
||||
|
||||
fn make_usage(input: u64) -> llm_engine::timeline::event::UsageEvent {
|
||||
llm_engine::timeline::event::UsageEvent {
|
||||
input_tokens: Some(input),
|
||||
output_tokens: Some(0),
|
||||
total_tokens: Some(input),
|
||||
cache_read_input_tokens: None,
|
||||
cache_creation_input_tokens: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_worker_interceptor_uses_occupancy_not_cumulative_usage() {
|
||||
let tracker = Arc::new(UsageTracker::new());
|
||||
let interceptor = CompactWorkerInterceptor::new(tracker.clone(), 150, 0, 0, None);
|
||||
let mut context = vec![Item::user_message("hello")];
|
||||
|
||||
assert!(matches!(
|
||||
interceptor.pre_llm_request(&mut context).await,
|
||||
PreRequestAction::Continue
|
||||
));
|
||||
tracker.record_usage(&make_usage(100));
|
||||
|
||||
assert!(matches!(
|
||||
interceptor.pre_llm_request(&mut context).await,
|
||||
PreRequestAction::Continue
|
||||
));
|
||||
tracker.record_usage(&make_usage(100));
|
||||
|
||||
// Two 100-token requests would exceed a cumulative 150-token cap, but
|
||||
// current occupancy is still the latest 100-token measurement.
|
||||
assert!(matches!(
|
||||
interceptor.pre_llm_request(&mut context).await,
|
||||
PreRequestAction::Continue
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_worker_interceptor_warns_before_hard_cap() {
|
||||
let tracker = Arc::new(UsageTracker::new());
|
||||
let warnings = Arc::new(Mutex::new(Vec::new()));
|
||||
let captured = warnings.clone();
|
||||
let interceptor = CompactWorkerInterceptor::new(
|
||||
tracker.clone(),
|
||||
150,
|
||||
60,
|
||||
20,
|
||||
Some(Arc::new(move |message| {
|
||||
captured.lock().unwrap().push(message);
|
||||
})),
|
||||
);
|
||||
let mut context = vec![Item::user_message("hello")];
|
||||
|
||||
assert!(matches!(
|
||||
interceptor.pre_llm_request(&mut context).await,
|
||||
PreRequestAction::Continue
|
||||
));
|
||||
tracker.record_usage(&make_usage(100));
|
||||
|
||||
assert!(matches!(
|
||||
interceptor.pre_llm_request(&mut context).await,
|
||||
PreRequestAction::ContinueWith(items)
|
||||
if items.len() == 1 && items[0].as_text().unwrap_or_default().contains("write_summary")
|
||||
));
|
||||
assert_eq!(warnings.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compact_worker_interceptor_cancels_when_occupancy_exceeds_cap() {
|
||||
let tracker = Arc::new(UsageTracker::new());
|
||||
let interceptor = CompactWorkerInterceptor::new(tracker.clone(), 99, 0, 0, None);
|
||||
let mut context = vec![Item::user_message("hello")];
|
||||
|
||||
assert!(matches!(
|
||||
interceptor.pre_llm_request(&mut context).await,
|
||||
PreRequestAction::Continue
|
||||
));
|
||||
tracker.record_usage(&make_usage(100));
|
||||
|
||||
assert!(matches!(
|
||||
interceptor.pre_llm_request(&mut context).await,
|
||||
PreRequestAction::Cancel(message) if message.contains("occupancy")
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mark_read_required_records_and_deducts_budget() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let path = tmp.path().join("hello.txt");
|
||||
std::fs::write(&path, "hello world\n").unwrap();
|
||||
|
||||
let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(1_000)));
|
||||
let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool {
|
||||
fs: make_fs(tmp.path()),
|
||||
ctx: ctx.clone(),
|
||||
});
|
||||
let input = serde_json::json!({ "file_path": path.to_str().unwrap() }).to_string();
|
||||
let out = tool.execute(&input, Default::default()).await.unwrap();
|
||||
|
||||
assert!(out.summary.starts_with("Marked"));
|
||||
let guard = ctx.lock().unwrap();
|
||||
assert_eq!(guard.read_required.len(), 1);
|
||||
assert!(guard.auto_read_consumed > 0);
|
||||
assert!(guard.auto_read_consumed <= 1_000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mark_read_required_rejects_over_budget() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let path = tmp.path().join("big.txt");
|
||||
std::fs::write(&path, "x".repeat(4_096)).unwrap(); // ≈1024 tokens
|
||||
|
||||
let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(100)));
|
||||
let tool: Arc<dyn Tool> = Arc::new(MarkReadRequiredTool {
|
||||
fs: make_fs(tmp.path()),
|
||||
ctx: ctx.clone(),
|
||||
});
|
||||
let input = serde_json::json!({ "file_path": path.to_str().unwrap() }).to_string();
|
||||
let res = tool.execute(&input, Default::default()).await;
|
||||
|
||||
assert!(matches!(res, Err(ToolError::ExecutionFailed(_))));
|
||||
let guard = ctx.lock().unwrap();
|
||||
assert!(guard.read_required.is_empty());
|
||||
assert_eq!(guard.auto_read_consumed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_summary_overwrites_previous_call() {
|
||||
let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(0)));
|
||||
let tool: Arc<dyn Tool> = Arc::new(WriteSummaryTool { ctx: ctx.clone() });
|
||||
|
||||
let first = serde_json::json!({ "text": "first" }).to_string();
|
||||
let out1 = tool.execute(&first, Default::default()).await.unwrap();
|
||||
assert!(out1.summary.contains("recorded"));
|
||||
|
||||
let second = serde_json::json!({ "text": "second" }).to_string();
|
||||
let out2 = tool.execute(&second, Default::default()).await.unwrap();
|
||||
assert!(out2.summary.contains("replaced"));
|
||||
|
||||
assert_eq!(ctx.lock().unwrap().summary.as_deref(), Some("second"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn add_reference_deduplicates() {
|
||||
let ctx = Arc::new(Mutex::new(CompactWorkerContext::with_budget(0)));
|
||||
let tool: Arc<dyn Tool> = Arc::new(AddReferenceTool { ctx: ctx.clone() });
|
||||
|
||||
let p = "/abs/path.rs";
|
||||
let input = serde_json::json!({ "file_path": p }).to_string();
|
||||
tool.execute(&input, Default::default()).await.unwrap();
|
||||
tool.execute(&input, Default::default()).await.unwrap();
|
||||
|
||||
let guard = ctx.lock().unwrap();
|
||||
assert_eq!(guard.references.len(), 1);
|
||||
assert_eq!(guard.references[0], PathBuf::from(p));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_session_log_returns_bounded_hits_without_full_tool_content() {
|
||||
let items = Arc::new(vec![
|
||||
Item::user_message("investigate compact failure"),
|
||||
Item::tool_result_with_content(
|
||||
"call-1",
|
||||
"read trace with compact failure",
|
||||
"very large raw trace body with secret detail",
|
||||
),
|
||||
]);
|
||||
let tool: Arc<dyn Tool> = Arc::new(SearchSessionLogTool {
|
||||
state: Arc::new(SessionLogToolState { items }),
|
||||
});
|
||||
let input = serde_json::json!({ "query": "compact", "limit": 10 }).to_string();
|
||||
let out = tool.execute(&input, Default::default()).await.unwrap();
|
||||
let content = out.content.unwrap();
|
||||
|
||||
assert!(content.contains("investigate compact failure"));
|
||||
assert!(content.contains("read trace with compact failure"));
|
||||
assert!(!content.contains("secret detail"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_session_items_full_mode_can_read_tool_result_content() {
|
||||
let items = Arc::new(vec![Item::tool_result_with_content(
|
||||
"call-1",
|
||||
"read trace",
|
||||
"raw trace detail",
|
||||
)]);
|
||||
let tool: Arc<dyn Tool> = Arc::new(ReadSessionItemsTool {
|
||||
state: Arc::new(SessionLogToolState { items }),
|
||||
});
|
||||
let input = serde_json::json!({ "offset": 0, "limit": 1, "mode": "full" }).to_string();
|
||||
let out = tool.execute(&input, Default::default()).await.unwrap();
|
||||
let content = out.content.unwrap();
|
||||
|
||||
assert!(content.contains("raw trace detail"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slice_lines_handles_offset_and_limit() {
|
||||
let text = "a\nb\nc\nd";
|
||||
assert_eq!(slice_lines(text, 0, None), "a\nb\nc\nd");
|
||||
assert_eq!(slice_lines(text, 1, Some(2)), "b\nc");
|
||||
assert_eq!(slice_lines(text, 10, None), "");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user