Merge remote-tracking branch 'origin/develop' into work/T-673-subjektiv-session-history
This commit is contained in:
@@ -151,7 +151,7 @@ impl WorkerHandle {
|
||||
.snapshot();
|
||||
let event = Event::Snapshot {
|
||||
session,
|
||||
greeting: self.shared_state.greeting.clone(),
|
||||
greeting: self.shared_state.greeting(),
|
||||
state: self.shared_state.snapshot(),
|
||||
in_flight,
|
||||
internal_workers: self.spawned_registry.internal_worker_snapshots(),
|
||||
@@ -269,6 +269,17 @@ async fn set_controller_state(
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn publish_context_usage(
|
||||
shared_state: &Arc<WorkerSharedState>,
|
||||
working_event_tx: &broadcast::Sender<Event>,
|
||||
tokens: Option<u64>,
|
||||
source: protocol::ContextTokenSource,
|
||||
) {
|
||||
shared_state.update_context_usage(tokens, source);
|
||||
let usage = tokens.map(|tokens| protocol::ContextUsage { tokens, source });
|
||||
let _ = working_event_tx.send(Event::ContextUsage { usage });
|
||||
}
|
||||
|
||||
async fn set_controller_status(
|
||||
shared_state: &Arc<WorkerSharedState>,
|
||||
runtime_dir: &RuntimeDir,
|
||||
@@ -303,6 +314,12 @@ async fn finish_controller_run<C, St>(
|
||||
// the terminal run boundary so reconnect snapshots cannot append stale
|
||||
// partial text/tool arguments after newer entries.
|
||||
worker.clear_in_flight_events();
|
||||
publish_context_usage(
|
||||
shared_state,
|
||||
working_event_tx,
|
||||
Some(worker.total_tokens().tokens),
|
||||
protocol::ContextTokenSource::Estimated,
|
||||
);
|
||||
set_controller_status(shared_state, runtime_dir, working_event_tx, new_status).await;
|
||||
}
|
||||
|
||||
@@ -792,6 +809,16 @@ impl WorkerController {
|
||||
manifest_toml.clone(),
|
||||
greeting,
|
||||
));
|
||||
let usage_state = shared_state.clone();
|
||||
let usage_events = working_event_tx.clone();
|
||||
worker.engine_mut().on_usage(move |event| {
|
||||
publish_context_usage(
|
||||
&usage_state,
|
||||
&usage_events,
|
||||
event.input_tokens,
|
||||
protocol::ContextTokenSource::Measured,
|
||||
);
|
||||
});
|
||||
if let Some(fs_for_view) = fs_for_view {
|
||||
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view));
|
||||
}
|
||||
@@ -2518,6 +2545,12 @@ async fn controller_loop<C, St>(
|
||||
}
|
||||
}
|
||||
};
|
||||
publish_context_usage(
|
||||
&shared_state,
|
||||
&working_event_tx,
|
||||
Some(worker.total_tokens().tokens),
|
||||
protocol::ContextTokenSource::Estimated,
|
||||
);
|
||||
if !matches!(
|
||||
result,
|
||||
Err(WorkerError::Store(_))
|
||||
@@ -2585,6 +2618,12 @@ async fn controller_loop<C, St>(
|
||||
.await
|
||||
{
|
||||
worker.clear_in_flight_events();
|
||||
publish_context_usage(
|
||||
&shared_state,
|
||||
&working_event_tx,
|
||||
Some(worker.total_tokens().tokens),
|
||||
protocol::ContextTokenSource::Estimated,
|
||||
);
|
||||
let snapshot = shared_state.transition(WorkerState::Idle);
|
||||
let _ = working_event_tx.send(Event::WorkerState { snapshot });
|
||||
}
|
||||
@@ -3392,7 +3431,24 @@ fn model_supports_image_attachments(model: &manifest::ModelManifest) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
fn build_greeting<C, St>(worker: &Worker<C, St>) -> protocol::Greeting
|
||||
fn public_reasoning_config(
|
||||
reasoning: &agen::llm_client::capability::ReasoningControl,
|
||||
) -> protocol::ReasoningConfig {
|
||||
match reasoning {
|
||||
agen::llm_client::capability::ReasoningControl::Effort(effort) => {
|
||||
protocol::ReasoningConfig::Effort {
|
||||
effort: effort.as_str().to_owned(),
|
||||
}
|
||||
}
|
||||
agen::llm_client::capability::ReasoningControl::BudgetTokens(budget_tokens) => {
|
||||
protocol::ReasoningConfig::BudgetTokens {
|
||||
budget_tokens: *budget_tokens,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_greeting<C, St>(worker: &Worker<C, St>) -> protocol::Greeting
|
||||
where
|
||||
C: LlmClient + 'static,
|
||||
St: Store,
|
||||
@@ -3435,6 +3491,7 @@ where
|
||||
.into_iter()
|
||||
.map(|def| def.name)
|
||||
.collect();
|
||||
let context_tokens = worker.total_tokens().tokens;
|
||||
protocol::Greeting {
|
||||
worker_name: manifest.worker.name.clone(),
|
||||
cwd: worker
|
||||
@@ -3443,10 +3500,19 @@ where
|
||||
.unwrap_or_default(),
|
||||
provider: provider_name,
|
||||
model: model_id,
|
||||
reasoning: manifest
|
||||
.engine
|
||||
.reasoning
|
||||
.as_ref()
|
||||
.map(public_reasoning_config),
|
||||
scope_summary: worker.scope_snapshot().summary(),
|
||||
tools: tool_names,
|
||||
context_window,
|
||||
context_tokens: worker.total_tokens().tokens,
|
||||
context_tokens,
|
||||
context_usage: Some(protocol::ContextUsage {
|
||||
tokens: context_tokens,
|
||||
source: protocol::ContextTokenSource::Estimated,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3509,6 +3575,69 @@ mod tests {
|
||||
assert!(materialize < exposure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_reasoning_config_preserves_effort_and_signed_budget() {
|
||||
let effort =
|
||||
public_reasoning_config(&agen::llm_client::capability::ReasoningControl::Effort(
|
||||
agen::llm_client::capability::ReasoningEffort::Other("provider-native".into()),
|
||||
));
|
||||
assert_eq!(
|
||||
effort,
|
||||
protocol::ReasoningConfig::Effort {
|
||||
effort: "provider-native".into(),
|
||||
}
|
||||
);
|
||||
|
||||
let budget = public_reasoning_config(
|
||||
&agen::llm_client::capability::ReasoningControl::BudgetTokens(-1),
|
||||
);
|
||||
assert_eq!(
|
||||
budget,
|
||||
protocol::ReasoningConfig::BudgetTokens { budget_tokens: -1 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn context_usage_publication_updates_reconnect_state_and_live_clients() {
|
||||
let shared_state = Arc::new(WorkerSharedState::new(
|
||||
"worker".into(),
|
||||
session_store::new_segment_id(),
|
||||
String::new(),
|
||||
protocol::Greeting {
|
||||
worker_name: "worker".into(),
|
||||
cwd: "/tmp".into(),
|
||||
provider: "provider".into(),
|
||||
model: "model".into(),
|
||||
reasoning: None,
|
||||
scope_summary: String::new(),
|
||||
tools: Vec::new(),
|
||||
context_window: 100_000,
|
||||
context_tokens: 0,
|
||||
context_usage: None,
|
||||
},
|
||||
));
|
||||
let (events, mut receiver) = broadcast::channel(4);
|
||||
|
||||
publish_context_usage(
|
||||
&shared_state,
|
||||
&events,
|
||||
Some(25_000),
|
||||
protocol::ContextTokenSource::Estimated,
|
||||
);
|
||||
|
||||
let expected = protocol::ContextUsage {
|
||||
tokens: 25_000,
|
||||
source: protocol::ContextTokenSource::Estimated,
|
||||
};
|
||||
assert_eq!(shared_state.greeting().context_usage, Some(expected));
|
||||
assert!(matches!(
|
||||
receiver.try_recv(),
|
||||
Ok(Event::ContextUsage {
|
||||
usage: Some(usage)
|
||||
}) if usage == expected
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_memory_extraction_destinations_are_mutually_exclusive() {
|
||||
let mut memory = manifest::MemoryFeatureProfileConfig::default();
|
||||
@@ -3667,6 +3796,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
));
|
||||
let notify_buffer = NotifyBuffer::new();
|
||||
@@ -3716,6 +3847,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
@@ -4424,6 +4557,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 1,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
);
|
||||
assert!(
|
||||
|
||||
@@ -1507,6 +1507,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
@@ -1543,6 +1545,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
@@ -1634,6 +1638,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
@@ -1661,6 +1667,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
@@ -1765,6 +1773,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Paused.into(),
|
||||
in_flight: Default::default(),
|
||||
@@ -1820,6 +1830,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
//! need durable domain audit must keep using their domain authority (for example Memory audit).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
|
||||
use agen::timeline::event::UsageEvent;
|
||||
use agen::{Engine, EngineError, llm_client::LlmClient};
|
||||
@@ -17,7 +17,9 @@ use session_store::{LogEntry, SegmentId, SessionId, Store, StoreError, TraceEntr
|
||||
use tokio::sync::broadcast;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::controller::{wire_event_bridges_on_engine, wire_workdir_command_events};
|
||||
use crate::controller::{
|
||||
build_greeting, wire_event_bridges_on_engine, wire_workdir_command_events,
|
||||
};
|
||||
use crate::feature::FeatureRegistryBuilder;
|
||||
use crate::in_flight::{InFlightEvents, snapshot_from_guard};
|
||||
use crate::ipc::alerter::Alerter;
|
||||
@@ -409,6 +411,7 @@ enum InternalWorkerSessionCommand {
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct InternalWorkerSessionSnapshot {
|
||||
pub session: protocol::SessionSnapshot,
|
||||
pub greeting: Option<protocol::Greeting>,
|
||||
pub status: WorkerStatus,
|
||||
pub error: Option<String>,
|
||||
pub in_flight: InFlightSnapshot,
|
||||
@@ -428,6 +431,7 @@ pub(crate) struct InternalWorkerSessionHandle {
|
||||
visibility: InternalWorkerVisibility,
|
||||
last_error: Arc<Mutex<Option<String>>>,
|
||||
last_outcome: Arc<Mutex<Option<InternalWorkerTurnOutcome>>>,
|
||||
greeting: Option<Arc<RwLock<protocol::Greeting>>>,
|
||||
child_registry: Option<Arc<SpawnedWorkerRegistry>>,
|
||||
sink: SegmentLogSink,
|
||||
#[cfg(test)]
|
||||
@@ -481,6 +485,12 @@ impl InternalWorkerSessionHandle {
|
||||
};
|
||||
InternalWorkerSessionSnapshot {
|
||||
session: session_store::public_snapshot::project_current_session_snapshot(&entries),
|
||||
greeting: self.greeting.as_ref().map(|greeting| {
|
||||
greeting
|
||||
.read()
|
||||
.expect("internal Worker greeting lock poisoned")
|
||||
.clone()
|
||||
}),
|
||||
status: match self.status() {
|
||||
InternalWorkerSessionStatus::Running => WorkerStatus::Running,
|
||||
InternalWorkerSessionStatus::Paused => WorkerStatus::Paused,
|
||||
@@ -824,6 +834,23 @@ fn spawn_internal_log_event_bridge(sink: SegmentLogSink, event_tx: broadcast::Se
|
||||
});
|
||||
}
|
||||
|
||||
fn publish_internal_context_usage(
|
||||
greeting: &RwLock<protocol::Greeting>,
|
||||
event_tx: &broadcast::Sender<Event>,
|
||||
tokens: Option<u64>,
|
||||
source: protocol::ContextTokenSource,
|
||||
) {
|
||||
let usage = tokens.map(|tokens| protocol::ContextUsage { tokens, source });
|
||||
{
|
||||
let mut greeting = greeting
|
||||
.write()
|
||||
.expect("internal Worker greeting lock poisoned");
|
||||
greeting.context_tokens = tokens.unwrap_or_default();
|
||||
greeting.context_usage = usage;
|
||||
}
|
||||
let _ = event_tx.send(Event::ContextUsage { usage });
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_internal_worker_session(
|
||||
mut worker: Worker<Box<dyn LlmClient>, EphemeralSessionStore>,
|
||||
store: EphemeralSessionStore,
|
||||
@@ -851,8 +878,20 @@ pub(crate) async fn prepare_internal_worker_session(
|
||||
worker.attach_alerter(alerter.clone());
|
||||
worker.attach_working_event_tx(event_tx.clone());
|
||||
worker.attach_in_flight_events(in_flight.clone());
|
||||
let greeting = Arc::new(RwLock::new(build_greeting(&worker)));
|
||||
wire_event_bridges_on_engine(&mut worker, &event_tx, &alerter, &in_flight);
|
||||
let usage_greeting = greeting.clone();
|
||||
let usage_events = event_tx.clone();
|
||||
worker.engine_mut().on_usage(move |event| {
|
||||
publish_internal_context_usage(
|
||||
&usage_greeting,
|
||||
&usage_events,
|
||||
event.input_tokens,
|
||||
protocol::ContextTokenSource::Measured,
|
||||
);
|
||||
});
|
||||
|
||||
let actor_greeting = greeting.clone();
|
||||
let session_id = worker.session_id();
|
||||
let segment_id = worker.segment_id();
|
||||
let (command_tx, mut command_rx) = tokio::sync::mpsc::channel(8);
|
||||
@@ -874,6 +913,7 @@ pub(crate) async fn prepare_internal_worker_session(
|
||||
visibility,
|
||||
last_error: last_error.clone(),
|
||||
last_outcome: last_outcome.clone(),
|
||||
greeting: Some(greeting),
|
||||
child_registry,
|
||||
sink,
|
||||
#[cfg(test)]
|
||||
@@ -886,50 +926,56 @@ pub(crate) async fn prepare_internal_worker_session(
|
||||
match command {
|
||||
InternalWorkerSessionCommand::Run(input) => {
|
||||
actor_in_flight.clear();
|
||||
let cancel_sender = worker.engine_mut().cancel_sender();
|
||||
let mut run = std::pin::pin!(worker.run_text(&input));
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut run => {
|
||||
let (turn_status, error, outcome) =
|
||||
classify_internal_turn_result(result);
|
||||
actor_in_flight.clear();
|
||||
*last_outcome.lock().unwrap() = Some(outcome);
|
||||
status.store(turn_status.encode(), std::sync::atomic::Ordering::Release);
|
||||
if let Some(message) = error {
|
||||
*last_error.lock().unwrap() = Some(message.clone());
|
||||
let _ = event_tx.send(Event::Error {
|
||||
code: protocol::ErrorCode::Internal,
|
||||
message,
|
||||
});
|
||||
let (turn_status, error, outcome) = {
|
||||
let cancel_sender = worker.engine_mut().cancel_sender();
|
||||
let mut run = std::pin::pin!(worker.run_text(&input));
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut run => {
|
||||
break classify_internal_turn_result(result);
|
||||
}
|
||||
send_internal_worker_state(&event_tx, turn_status);
|
||||
if let Some(callback) = &on_turn_end {
|
||||
callback(turn_status);
|
||||
}
|
||||
state_changed.notify_waiters();
|
||||
break;
|
||||
}
|
||||
command = command_rx.recv() => {
|
||||
match command {
|
||||
Some(InternalWorkerSessionCommand::Stop(done)) => {
|
||||
let _ = cancel_sender.send(()).await;
|
||||
let _ = (&mut run).await;
|
||||
stop_done = Some(done);
|
||||
break 'actor;
|
||||
}
|
||||
Some(InternalWorkerSessionCommand::Run(_)) => {
|
||||
// `send` reserves Running atomically, so a second Run cannot be enqueued.
|
||||
}
|
||||
None => {
|
||||
let _ = cancel_sender.send(()).await;
|
||||
let _ = (&mut run).await;
|
||||
break 'actor;
|
||||
command = command_rx.recv() => {
|
||||
match command {
|
||||
Some(InternalWorkerSessionCommand::Stop(done)) => {
|
||||
let _ = cancel_sender.send(()).await;
|
||||
let _ = (&mut run).await;
|
||||
stop_done = Some(done);
|
||||
break 'actor;
|
||||
}
|
||||
Some(InternalWorkerSessionCommand::Run(_)) => {
|
||||
// `send` reserves Running atomically, so a second Run cannot be enqueued.
|
||||
}
|
||||
None => {
|
||||
let _ = cancel_sender.send(()).await;
|
||||
let _ = (&mut run).await;
|
||||
break 'actor;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
publish_internal_context_usage(
|
||||
&actor_greeting,
|
||||
&event_tx,
|
||||
Some(worker.total_tokens().tokens),
|
||||
protocol::ContextTokenSource::Estimated,
|
||||
);
|
||||
actor_in_flight.clear();
|
||||
*last_outcome.lock().unwrap() = Some(outcome);
|
||||
status.store(turn_status.encode(), std::sync::atomic::Ordering::Release);
|
||||
if let Some(message) = error {
|
||||
*last_error.lock().unwrap() = Some(message.clone());
|
||||
let _ = event_tx.send(Event::Error {
|
||||
code: protocol::ErrorCode::Internal,
|
||||
message,
|
||||
});
|
||||
}
|
||||
send_internal_worker_state(&event_tx, turn_status);
|
||||
if let Some(callback) = &on_turn_end {
|
||||
callback(turn_status);
|
||||
}
|
||||
state_changed.notify_waiters();
|
||||
}
|
||||
InternalWorkerSessionCommand::Stop(done) => {
|
||||
stop_done = Some(done);
|
||||
@@ -1174,6 +1220,14 @@ impl session_store::WorkerMetadataStore for EphemeralSessionStore {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_internal_worker_session(
|
||||
visibility: InternalWorkerVisibility,
|
||||
) -> (InternalWorkerSessionHandle, broadcast::Sender<Event>) {
|
||||
test_internal_worker_session_with_greeting(visibility, None)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_internal_worker_session_with_greeting(
|
||||
visibility: InternalWorkerVisibility,
|
||||
greeting: Option<protocol::Greeting>,
|
||||
) -> (InternalWorkerSessionHandle, broadcast::Sender<Event>) {
|
||||
let store = EphemeralSessionStore::default();
|
||||
let session_id = session_store::new_session_id();
|
||||
@@ -1206,6 +1260,7 @@ pub(crate) fn test_internal_worker_session(
|
||||
visibility,
|
||||
last_error: Arc::new(Mutex::new(None)),
|
||||
last_outcome: Arc::new(Mutex::new(None)),
|
||||
greeting: greeting.map(|greeting| Arc::new(RwLock::new(greeting))),
|
||||
child_registry: None,
|
||||
sink,
|
||||
fail_stop: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
@@ -1485,11 +1540,37 @@ permission = "write"
|
||||
assert_eq!(handle.protocol_snapshot().in_flight.blocks.len(), 1);
|
||||
let entries_after_first = handle.entries().len();
|
||||
assert!(entries_after_first >= 4);
|
||||
assert!(matches!(
|
||||
handle
|
||||
.protocol_snapshot()
|
||||
.greeting
|
||||
.and_then(|greeting| greeting.context_usage),
|
||||
Some(protocol::ContextUsage {
|
||||
source: protocol::ContextTokenSource::Estimated,
|
||||
..
|
||||
})
|
||||
));
|
||||
let mut events = handle.subscribe_events();
|
||||
handle.send("follow-up").await.expect("send follow-up turn");
|
||||
assert_eq!(
|
||||
handle.wait_until_idle().await,
|
||||
InternalWorkerSessionStatus::Idle
|
||||
);
|
||||
let mut saw_estimated_context = false;
|
||||
while let Ok(event) = events.try_recv() {
|
||||
if matches!(
|
||||
event,
|
||||
Event::ContextUsage {
|
||||
usage: Some(protocol::ContextUsage {
|
||||
source: protocol::ContextTokenSource::Estimated,
|
||||
..
|
||||
})
|
||||
}
|
||||
) {
|
||||
saw_estimated_context = true;
|
||||
}
|
||||
}
|
||||
assert!(saw_estimated_context);
|
||||
assert!(handle.protocol_snapshot().in_flight.blocks.is_empty());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
assert!(handle.entries().len() > entries_after_first);
|
||||
|
||||
@@ -16,9 +16,23 @@ pub struct WorkerProtocolSessionStreams {
|
||||
pub events: broadcast::Receiver<Event>,
|
||||
}
|
||||
|
||||
fn subscribe_before_snapshot<Subscription, Snapshot>(
|
||||
subscribe: impl FnOnce() -> Subscription,
|
||||
snapshot: impl FnOnce() -> Snapshot,
|
||||
) -> (Snapshot, Subscription) {
|
||||
// Subscription must exist before any snapshot field is read. A concurrent
|
||||
// state update can then appear in both lanes, but it can never be absent
|
||||
// from both the snapshot and the queued live stream.
|
||||
let subscription = subscribe();
|
||||
let snapshot = snapshot();
|
||||
(snapshot, subscription)
|
||||
}
|
||||
|
||||
pub fn subscribe_worker_protocol_session(handle: &WorkerHandle) -> WorkerProtocolSessionStreams {
|
||||
let (snapshot_event, log_entries) = handle.snapshot_event_with_entry_subscription();
|
||||
let (alert_snapshot, events) = handle.alerter.subscribe_with_snapshot();
|
||||
let ((snapshot_event, log_entries), (alert_snapshot, events)) = subscribe_before_snapshot(
|
||||
|| handle.alerter.subscribe_with_snapshot(),
|
||||
|| handle.snapshot_event_with_entry_subscription(),
|
||||
);
|
||||
WorkerProtocolSessionStreams {
|
||||
snapshot_event,
|
||||
alert_snapshot,
|
||||
@@ -129,6 +143,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn context_publication_between_subscription_and_snapshot_cannot_be_missed() {
|
||||
let (events, _) = broadcast::channel(4);
|
||||
let context_tokens = std::cell::Cell::new(20_u64);
|
||||
|
||||
let (snapshot_tokens, mut receiver) = subscribe_before_snapshot(
|
||||
|| events.subscribe(),
|
||||
|| {
|
||||
context_tokens.set(25);
|
||||
events
|
||||
.send(Event::ContextUsage {
|
||||
usage: Some(protocol::ContextUsage {
|
||||
tokens: 25,
|
||||
source: protocol::ContextTokenSource::Estimated,
|
||||
}),
|
||||
})
|
||||
.unwrap();
|
||||
context_tokens.get()
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(snapshot_tokens, 25);
|
||||
assert!(matches!(
|
||||
receiver.try_recv(),
|
||||
Ok(Event::ContextUsage {
|
||||
usage: Some(protocol::ContextUsage { tokens: 25, .. })
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persisted_run_transitions_keep_identity_across_live_and_snapshot_projection() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -212,6 +212,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ pub struct WorkerSharedState {
|
||||
pub worker_name: String,
|
||||
pub segment_id: SegmentId,
|
||||
pub manifest_toml: String,
|
||||
pub greeting: protocol::Greeting,
|
||||
greeting: RwLock<protocol::Greeting>,
|
||||
state: RwLock<WorkerStateSnapshot>,
|
||||
accepted_commands: RwLock<VecDeque<AcceptedWorkerCommand>>,
|
||||
/// Worker-from-the-inside view of the filesystem. Set once in
|
||||
@@ -62,7 +62,7 @@ impl WorkerSharedState {
|
||||
worker_name,
|
||||
segment_id,
|
||||
manifest_toml,
|
||||
greeting,
|
||||
greeting: RwLock::new(greeting),
|
||||
state: RwLock::new(WorkerStateSnapshot::initial()),
|
||||
accepted_commands: RwLock::new(VecDeque::new()),
|
||||
fs_view: OnceLock::new(),
|
||||
@@ -90,6 +90,22 @@ impl WorkerSharedState {
|
||||
self.flow_transition_enabled.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn greeting(&self) -> protocol::Greeting {
|
||||
self.greeting
|
||||
.read()
|
||||
.expect("worker greeting lock poisoned")
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub fn update_context_usage(&self, tokens: Option<u64>, source: protocol::ContextTokenSource) {
|
||||
let mut greeting = self
|
||||
.greeting
|
||||
.write()
|
||||
.expect("worker greeting lock poisoned");
|
||||
greeting.context_tokens = tokens.unwrap_or_default();
|
||||
greeting.context_usage = tokens.map(|tokens| protocol::ContextUsage { tokens, source });
|
||||
}
|
||||
|
||||
pub fn transition(&self, state: WorkerState) -> WorkerStateSnapshot {
|
||||
let mut snapshot = self
|
||||
.state
|
||||
@@ -245,9 +261,32 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_context_usage_is_reflected_in_reconnect_greeting() {
|
||||
let state = test_state();
|
||||
state.update_context_usage(Some(42_000), protocol::ContextTokenSource::Measured);
|
||||
|
||||
let greeting = state.greeting();
|
||||
assert_eq!(greeting.context_tokens, 42_000);
|
||||
assert_eq!(
|
||||
greeting.context_usage,
|
||||
Some(protocol::ContextUsage {
|
||||
tokens: 42_000,
|
||||
source: protocol::ContextTokenSource::Measured,
|
||||
})
|
||||
);
|
||||
|
||||
state.update_context_usage(None, protocol::ContextTokenSource::Measured);
|
||||
let greeting = state.greeting();
|
||||
assert_eq!(greeting.context_tokens, 0);
|
||||
assert_eq!(greeting.context_usage, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initial_snapshot_is_idle() {
|
||||
let state = test_state();
|
||||
|
||||
@@ -97,6 +97,8 @@ mod tests {
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
|
||||
@@ -29,7 +29,9 @@ use tokio::sync::{Notify, broadcast};
|
||||
use tracing::warn;
|
||||
use workdir::WorkdirScopeLeaseSet;
|
||||
|
||||
use crate::internal_worker::{InternalWorkerSessionHandle, InternalWorkerVisibility};
|
||||
use crate::internal_worker::{
|
||||
InternalWorkerSessionHandle, InternalWorkerSessionSnapshot, InternalWorkerVisibility,
|
||||
};
|
||||
use crate::runtime::dir::{RuntimeDir, SpawnedWorkerRecord};
|
||||
use crate::runtime::worker_allocation;
|
||||
|
||||
@@ -648,6 +650,14 @@ impl SpawnedWorkerRegistry {
|
||||
let protocol_emit_lock = record.protocol_emit_lock.clone();
|
||||
let protocol_terminal = record.protocol_terminal.clone();
|
||||
let mut child_rx = record.session.subscribe_events();
|
||||
publish_initial_internal_snapshot(
|
||||
&parent_tx,
|
||||
&worker,
|
||||
&protocol_revision,
|
||||
&protocol_emit_lock,
|
||||
&protocol_terminal,
|
||||
&record.session,
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match child_rx.recv().await {
|
||||
@@ -710,6 +720,14 @@ impl SpawnedWorkerRegistry {
|
||||
let protocol_emit_lock = record.protocol_emit_lock.clone();
|
||||
let protocol_terminal = record.protocol_terminal.clone();
|
||||
let mut child_rx = record.session.subscribe_events();
|
||||
publish_initial_internal_snapshot(
|
||||
&parent_tx,
|
||||
&worker,
|
||||
&protocol_revision,
|
||||
&protocol_emit_lock,
|
||||
&protocol_terminal,
|
||||
&record.session,
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match child_rx.recv().await {
|
||||
@@ -985,6 +1003,41 @@ impl SpawnedWorkerRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
fn publish_initial_internal_snapshot(
|
||||
parent_tx: &broadcast::Sender<Event>,
|
||||
worker: &InternalWorkerRef,
|
||||
protocol_revision: &AtomicU64,
|
||||
protocol_emit_lock: &Mutex<()>,
|
||||
protocol_terminal: &AtomicBool,
|
||||
session: &InternalWorkerSessionHandle,
|
||||
) {
|
||||
let Some(event) = internal_worker_live_snapshot(session.protocol_snapshot()) else {
|
||||
return;
|
||||
};
|
||||
let _emit_guard = protocol_emit_lock
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
if protocol_terminal.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let revision = protocol_revision.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
let _ = parent_tx.send(Event::InternalWorker {
|
||||
worker: worker.clone(),
|
||||
revision,
|
||||
event: Box::new(event),
|
||||
});
|
||||
}
|
||||
|
||||
fn internal_worker_live_snapshot(snapshot: InternalWorkerSessionSnapshot) -> Option<Event> {
|
||||
Some(Event::Snapshot {
|
||||
session: snapshot.session,
|
||||
greeting: snapshot.greeting?,
|
||||
state: snapshot.status.into(),
|
||||
in_flight: snapshot.in_flight,
|
||||
internal_workers: snapshot.internal_workers,
|
||||
})
|
||||
}
|
||||
|
||||
fn internal_worker_snapshot(
|
||||
worker: InternalWorkerRef,
|
||||
revision: u64,
|
||||
@@ -995,6 +1048,7 @@ fn internal_worker_snapshot(
|
||||
worker,
|
||||
revision,
|
||||
session: snapshot.session,
|
||||
greeting: snapshot.greeting,
|
||||
status: snapshot.status,
|
||||
error: snapshot.error,
|
||||
in_flight: snapshot.in_flight,
|
||||
@@ -1121,7 +1175,10 @@ mod tests {
|
||||
use session_store::LogEntry;
|
||||
|
||||
use super::*;
|
||||
use crate::internal_worker::{InternalWorkerSessionStatus, test_internal_worker_session};
|
||||
use crate::internal_worker::{
|
||||
InternalWorkerSessionStatus, test_internal_worker_session,
|
||||
test_internal_worker_session_with_greeting,
|
||||
};
|
||||
|
||||
fn registry() -> Arc<SpawnedWorkerRegistry> {
|
||||
let scope = Scope::from_config(&ScopeConfig {
|
||||
@@ -1141,7 +1198,15 @@ mod tests {
|
||||
name: &str,
|
||||
visibility: InternalWorkerVisibility,
|
||||
) -> (InternalSpawnedWorkerRecord, broadcast::Sender<Event>) {
|
||||
let (session, sender) = test_internal_worker_session(visibility);
|
||||
record_with_greeting(name, visibility, None).await
|
||||
}
|
||||
|
||||
async fn record_with_greeting(
|
||||
name: &str,
|
||||
visibility: InternalWorkerVisibility,
|
||||
greeting: Option<protocol::Greeting>,
|
||||
) -> (InternalSpawnedWorkerRecord, broadcast::Sender<Event>) {
|
||||
let (session, sender) = test_internal_worker_session_with_greeting(visibility, greeting);
|
||||
let root = std::path::PathBuf::from("/tmp");
|
||||
let scope = Scope::from_config(&ScopeConfig {
|
||||
allow: vec![ScopeRule {
|
||||
@@ -1193,6 +1258,95 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn visible_internal_worker_starts_with_own_live_snapshot_metadata() {
|
||||
let registry = registry();
|
||||
let (parent_tx, mut parent_rx) = broadcast::channel(16);
|
||||
registry.attach_parent_protocol(parent_tx, "parent-session".into());
|
||||
let greeting = protocol::Greeting {
|
||||
worker_name: "research".into(),
|
||||
cwd: "/child".into(),
|
||||
provider: "provider".into(),
|
||||
model: "child-model".into(),
|
||||
reasoning: Some(protocol::ReasoningConfig::Effort {
|
||||
effort: "high".into(),
|
||||
}),
|
||||
scope_summary: "child scope".into(),
|
||||
tools: Vec::new(),
|
||||
context_window: 64_000,
|
||||
context_tokens: 12_000,
|
||||
context_usage: Some(protocol::ContextUsage {
|
||||
tokens: 12_000,
|
||||
source: protocol::ContextTokenSource::Measured,
|
||||
}),
|
||||
};
|
||||
let (record, child_tx) = record_with_greeting(
|
||||
"research",
|
||||
InternalWorkerVisibility::ParentClient,
|
||||
Some(greeting),
|
||||
)
|
||||
.await;
|
||||
registry
|
||||
.internal_records
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(record.clone());
|
||||
|
||||
registry.start_protocol_forwarding(record);
|
||||
|
||||
let initial = tokio::time::timeout(Duration::from_secs(1), parent_rx.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
initial,
|
||||
Event::InternalWorker {
|
||||
revision: 1,
|
||||
event,
|
||||
..
|
||||
} if matches!(
|
||||
*event,
|
||||
Event::Snapshot {
|
||||
greeting: protocol::Greeting {
|
||||
ref model,
|
||||
context_window: 64_000,
|
||||
..
|
||||
},
|
||||
..
|
||||
} if model == "child-model"
|
||||
)
|
||||
));
|
||||
|
||||
child_tx
|
||||
.send(Event::ContextUsage {
|
||||
usage: Some(protocol::ContextUsage {
|
||||
tokens: 14_000,
|
||||
source: protocol::ContextTokenSource::Estimated,
|
||||
}),
|
||||
})
|
||||
.unwrap();
|
||||
let update = tokio::time::timeout(Duration::from_secs(1), parent_rx.recv())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
update,
|
||||
Event::InternalWorker {
|
||||
revision: 2,
|
||||
event,
|
||||
..
|
||||
} if matches!(
|
||||
*event,
|
||||
Event::ContextUsage {
|
||||
usage: Some(protocol::ContextUsage {
|
||||
tokens: 14_000,
|
||||
source: protocol::ContextTokenSource::Estimated,
|
||||
})
|
||||
}
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn visible_internal_output_is_wrapped_after_registry_insertion() {
|
||||
let registry = registry();
|
||||
|
||||
Reference in New Issue
Block a user