feat: integrate Worker state authority

This commit is contained in:
2026-09-06 11:00:00 +09:00
43 changed files with 3127 additions and 993 deletions
+3 -5
View File
@@ -112,8 +112,8 @@ mod tests {
async fn encodes_methods_and_decodes_events_above_transport() { async fn encodes_methods_and_decodes_events_above_transport() {
let mut socket = TestSocket::default(); let mut socket = TestSocket::default();
socket.incoming.push_back( socket.incoming.push_back(
encode_event(&Event::Status { encode_event(&Event::WorkerState {
status: WorkerStatus::Idle, snapshot: WorkerStatus::Idle.into(),
}) })
.expect("encode event"), .expect("encode event"),
); );
@@ -132,9 +132,7 @@ mod tests {
)); ));
assert!(matches!( assert!(matches!(
client.next_event().await, client.next_event().await,
Ok(Some(Event::Status { Ok(Some(Event::WorkerState { .. }))
status: WorkerStatus::Idle
}))
)); ));
} }
} }
+3 -5
View File
@@ -101,8 +101,8 @@ mod tests {
)); ));
peer.send( peer.send(
encode_event(&Event::Status { encode_event(&Event::WorkerState {
status: WorkerStatus::Idle, snapshot: WorkerStatus::Idle.into(),
}) })
.expect("encode event"), .expect("encode event"),
) )
@@ -110,9 +110,7 @@ mod tests {
.expect("send event"); .expect("send event");
assert!(matches!( assert!(matches!(
client.next_event().await, client.next_event().await,
Ok(Some(Event::Status { Ok(Some(Event::WorkerState { .. }))
status: WorkerStatus::Idle
}))
)); ));
} }
} }
+3 -8
View File
@@ -113,8 +113,8 @@ mod tests {
let listener = UnixListener::bind(&socket_path).unwrap(); let listener = UnixListener::bind(&socket_path).unwrap();
let server = tokio::spawn(async move { let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap(); let (mut stream, _) = listener.accept().await.unwrap();
let event = encode_event(&Event::Status { let event = encode_event(&Event::WorkerState {
status: WorkerStatus::Idle, snapshot: WorkerStatus::Idle.into(),
}) })
.unwrap(); .unwrap();
stream.write_all(event.as_bytes()).await.unwrap(); stream.write_all(event.as_bytes()).await.unwrap();
@@ -126,12 +126,7 @@ mod tests {
.await .await
.expect("client should receive event while alive") .expect("client should receive event while alive")
.expect("transport should succeed"); .expect("transport should succeed");
assert!(matches!( assert!(matches!(event, Some(Event::WorkerState { .. })));
event,
Some(Event::Status {
status: WorkerStatus::Idle
})
));
server.await.unwrap(); server.await.unwrap();
} }
+3 -5
View File
@@ -116,8 +116,8 @@ mod tests {
Message::Text(ref text) Message::Text(ref text)
if matches!(decode_method(text), Ok(Method::Submit { .. })) if matches!(decode_method(text), Ok(Method::Submit { .. }))
)); ));
let event = encode_event(&Event::Status { let event = encode_event(&Event::WorkerState {
status: WorkerStatus::Idle, snapshot: WorkerStatus::Idle.into(),
}) })
.unwrap(); .unwrap();
socket.send(Message::Text(event.into())).await.unwrap(); socket.send(Message::Text(event.into())).await.unwrap();
@@ -134,9 +134,7 @@ mod tests {
.expect("send method"); .expect("send method");
assert!(matches!( assert!(matches!(
client.next_event().await, client.next_event().await,
Ok(Some(Event::Status { Ok(Some(Event::WorkerState { .. }))
status: WorkerStatus::Idle
}))
)); ));
server.await.unwrap(); server.await.unwrap();
} }
+322 -52
View File
@@ -85,6 +85,190 @@ impl AuthenticatedInputSource {
} }
} }
/// Immutable identity and revision fence for one state-changing Worker command.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct WorkerCommandEnvelope {
/// Caller-owned sequence. A controller accepts command ids in strictly
/// increasing order for one execution generation.
pub command_id: u64,
pub expected_execution_generation: u64,
pub expected_worker_state_revision: u64,
}
impl WorkerCommandEnvelope {
pub fn for_snapshot(command_id: u64, snapshot: &WorkerStateSnapshot) -> Self {
Self {
command_id,
expected_execution_generation: snapshot.execution_generation,
expected_worker_state_revision: snapshot.revision,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum WorkerCommandKind {
Resume,
Cancel,
Pause,
Compact,
Shutdown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum WorkerCommandDisposition {
Accepted,
StaleExecutionGeneration,
StaleWorkerStateRevision,
StaleCommandId,
Conflict,
InvalidState,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct WorkerCommandAcknowledgement {
pub command_id: u64,
pub command: WorkerCommandKind,
pub disposition: WorkerCommandDisposition,
/// The complete authoritative state observed after command admission.
pub state: WorkerStateSnapshot,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "kind", content = "state", rename_all = "snake_case")]
pub enum WorkerState {
Idle,
Busy(WorkerBusyState),
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "kind", content = "state", rename_all = "snake_case")]
pub enum WorkerBusyState {
Run(WorkerRunState),
Maintenance(WorkerMaintenanceState),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum WorkerRunState {
Running,
Pausing,
Paused,
Cancelling,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum WorkerMaintenanceState {
Compacting,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct WorkerStateSnapshot {
pub execution_generation: u64,
pub revision: u64,
/// Highest lifecycle command id observed by this controller generation.
pub last_command_id: u64,
pub state: WorkerState,
}
impl WorkerStateSnapshot {
pub fn initial(execution_generation: u64) -> Self {
Self {
execution_generation,
revision: 0,
last_command_id: 0,
state: WorkerState::Idle,
}
}
/// Compatibility projection for Runtime catalog lifecycle. This value is
/// never command-admission authority and cannot produce `Stopped`.
pub fn catalog_status(&self) -> WorkerStatus {
match self.state {
WorkerState::Idle => WorkerStatus::Idle,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused,
WorkerState::Busy(WorkerBusyState::Run(_))
| WorkerState::Busy(WorkerBusyState::Maintenance(_)) => WorkerStatus::Running,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WorkerStateSnapshotApply {
Applied,
Duplicate,
Stale,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WorkerStateSnapshotConflict {
pub execution_generation: u64,
pub revision: u64,
}
impl std::fmt::Display for WorkerStateSnapshotConflict {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
formatter,
"conflicting worker state snapshots at generation {} revision {}",
self.execution_generation, self.revision
)
}
}
impl std::error::Error for WorkerStateSnapshotConflict {}
pub fn apply_worker_state_snapshot(
current: &mut WorkerStateSnapshot,
incoming: &WorkerStateSnapshot,
) -> Result<WorkerStateSnapshotApply, WorkerStateSnapshotConflict> {
use std::cmp::Ordering;
let ordering = (incoming.execution_generation, incoming.revision)
.cmp(&(current.execution_generation, current.revision));
match ordering {
Ordering::Greater => {
*current = incoming.clone();
Ok(WorkerStateSnapshotApply::Applied)
}
Ordering::Less => Ok(WorkerStateSnapshotApply::Stale),
Ordering::Equal if incoming == current => Ok(WorkerStateSnapshotApply::Duplicate),
Ordering::Equal => Err(WorkerStateSnapshotConflict {
execution_generation: incoming.execution_generation,
revision: incoming.revision,
}),
}
}
impl From<WorkerStatus> for WorkerStateSnapshot {
fn from(status: WorkerStatus) -> Self {
let state = match status {
WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
WorkerStatus::Running => {
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
}
WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
};
Self {
execution_generation: 1,
revision: 0,
last_command_id: 0,
state,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "method", content = "params", rename_all = "snake_case")] #[serde(tag = "method", content = "params", rename_all = "snake_case")]
@@ -149,20 +333,28 @@ pub enum Method {
expected_revision: u64, expected_revision: u64,
expected_head_id: String, expected_head_id: String,
}, },
Resume, Resume {
Cancel, command: WorkerCommandEnvelope,
},
Cancel {
command: WorkerCommandEnvelope,
},
/// Stop the in-flight turn and transition to `Paused`. /// Stop the in-flight turn and transition to `Paused`.
/// ///
/// Unlike `Cancel` (which discards and returns to `Idle`), a paused /// Unlike `Cancel` (which discards and returns to `Idle`), a paused
/// Worker can resume the interrupted work via `Resume`, or accept a /// Worker can resume the interrupted work via `Resume`, or accept a
/// fresh `Submit` (orphan `tool_use` items are closed with a /// fresh `Submit` (orphan `tool_use` items are closed with a
/// synthetic tool result before the new user message is appended). /// synthetic tool result before the new user message is appended).
Pause, Pause {
command: WorkerCommandEnvelope,
},
/// Request an explicit compaction while the Worker is otherwise idle. /// Request an explicit compaction while the Worker is otherwise idle.
/// ///
/// This is a typed control method: clients must not send `compact` as a /// This is a typed control method: clients must not send `compact` as a
/// `Method::Submit` user message. /// `Method::Submit` user message.
Compact, Compact {
command: WorkerCommandEnvelope,
},
/// Ask the Worker to list valid rewind targets from its authoritative session log. /// Ask the Worker to list valid rewind targets from its authoritative session log.
ListRewindTargets, ListRewindTargets,
/// Truncate the current session back to the selected rewind target and /// Truncate the current session back to the selected rewind target and
@@ -171,7 +363,9 @@ pub enum Method {
target: RewindTargetId, target: RewindTargetId,
expected_head_entries: usize, expected_head_entries: usize,
}, },
Shutdown, Shutdown {
command: WorkerCommandEnvelope,
},
/// Request a list of completion candidates from the Worker. /// Request a list of completion candidates from the Worker.
/// ///
/// Reply is sent on the same socket as `Event::Completions` (not /// Reply is sent on the same socket as `Event::Completions` (not
@@ -938,8 +1132,9 @@ pub enum Event {
Snapshot { Snapshot {
session: SessionSnapshot, session: SessionSnapshot,
greeting: Greeting, greeting: Greeting,
#[serde(default)] /// Full revisioned live execution state. `Stopped` remains Runtime
status: WorkerStatus, /// catalog authority and is deliberately not represented here.
state: WorkerStateSnapshot,
/// Unfinished model output that has already streamed in the current /// Unfinished model output that has already streamed in the current
/// run but is not yet represented by committed snapshot entries. /// run but is not yet represented by committed snapshot entries.
#[serde(default, skip_serializing_if = "InFlightSnapshot::is_empty")] #[serde(default, skip_serializing_if = "InFlightSnapshot::is_empty")]
@@ -976,8 +1171,11 @@ pub enum Event {
}, },
/// Current Worker controller status. Broadcast on every controller-level /// Current Worker controller status. Broadcast on every controller-level
/// transition and included in `History` snapshots for late attach. /// transition and included in `History` snapshots for late attach.
Status { WorkerState {
status: WorkerStatus, snapshot: WorkerStateSnapshot,
},
CommandAcknowledged {
acknowledgement: WorkerCommandAcknowledgement,
}, },
/// Bounded, provider-owned command telemetry for the live Console. This is /// Bounded, provider-owned command telemetry for the live Console. This is
/// intentionally not a history entry and is reconstructed from /// intentionally not a history entry and is reconstructed from
@@ -1424,6 +1622,58 @@ pub enum Permission {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn worker_state_snapshot_apply_is_monotonic_and_detects_conflicts() {
let mut current = WorkerStateSnapshot::initial(4);
let mut newer = current.clone();
newer.revision = 1;
newer.state = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running));
assert_eq!(
apply_worker_state_snapshot(&mut current, &newer),
Ok(WorkerStateSnapshotApply::Applied)
);
assert_eq!(
apply_worker_state_snapshot(&mut current, &newer),
Ok(WorkerStateSnapshotApply::Duplicate)
);
let stale_revision = WorkerStateSnapshot::initial(4);
assert_eq!(
apply_worker_state_snapshot(&mut current, &stale_revision),
Ok(WorkerStateSnapshotApply::Stale)
);
let stale_generation = WorkerStateSnapshot {
execution_generation: 3,
revision: u64::MAX,
..newer.clone()
};
assert_eq!(
apply_worker_state_snapshot(&mut current, &stale_generation),
Ok(WorkerStateSnapshotApply::Stale)
);
let conflicting = WorkerStateSnapshot {
state: WorkerState::Idle,
..newer.clone()
};
assert_eq!(
apply_worker_state_snapshot(&mut current, &conflicting),
Err(WorkerStateSnapshotConflict {
execution_generation: 4,
revision: 1,
})
);
assert_eq!(current, newer);
let next_generation = WorkerStateSnapshot::initial(5);
assert_eq!(
apply_worker_state_snapshot(&mut current, &next_generation),
Ok(WorkerStateSnapshotApply::Applied)
);
assert_eq!(current, next_generation);
}
#[test] #[test]
fn method_submit_json_roundtrip_and_run_is_rejected() { fn method_submit_json_roundtrip_and_run_is_rejected() {
let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#; let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#;
@@ -1612,28 +1862,39 @@ mod tests {
} }
#[test] #[test]
fn method_without_params() { fn lifecycle_method_without_command_fails_closed() {
let json = r#"{"method":"resume"}"#; let error = serde_json::from_str::<Method>(r#"{"method":"resume"}"#).unwrap_err();
let method: Method = serde_json::from_str(json).unwrap(); assert!(error.to_string().contains("params"));
assert!(matches!(method, Method::Resume));
} }
#[test] #[test]
fn method_pause_roundtrip() { fn lifecycle_methods_roundtrip_with_fences() {
let json = r#"{"method":"pause"}"#; for method in [
let method: Method = serde_json::from_str(json).unwrap(); Method::Pause {
assert!(matches!(method, Method::Pause)); command: WorkerCommandEnvelope {
let serialized = serde_json::to_string(&method).unwrap(); command_id: 11,
assert_eq!(serialized, json); expected_execution_generation: 4,
} expected_worker_state_revision: 8,
},
#[test] },
fn method_compact_roundtrip() { Method::Compact {
let json = r#"{"method":"compact"}"#; command: WorkerCommandEnvelope {
let method: Method = serde_json::from_str(json).unwrap(); command_id: 12,
assert!(matches!(method, Method::Compact)); expected_execution_generation: 4,
let serialized = serde_json::to_string(&method).unwrap(); expected_worker_state_revision: 9,
assert_eq!(serialized, json); },
},
] {
let json = serde_json::to_string(&method).unwrap();
let decoded: Method = serde_json::from_str(&json).unwrap();
match decoded {
Method::Pause { command } | Method::Compact { command } => {
assert_eq!(command.expected_execution_generation, 4);
assert!(command.command_id >= 11);
}
other => panic!("unexpected lifecycle method: {other:?}"),
}
}
} }
#[test] #[test]
@@ -1902,7 +2163,7 @@ mod tests {
context_window: 200_000, context_window: 200_000,
context_tokens: 42_000, context_tokens: 42_000,
}, },
status: WorkerStatus::Paused, state: WorkerStatus::Paused.into(),
in_flight: InFlightSnapshot::default(), in_flight: InFlightSnapshot::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}; };
@@ -1919,12 +2180,13 @@ mod tests {
assert_eq!(parsed["data"]["greeting"]["tools"][0], "Read"); assert_eq!(parsed["data"]["greeting"]["tools"][0], "Read");
assert_eq!(parsed["data"]["greeting"]["context_window"], 200_000); assert_eq!(parsed["data"]["greeting"]["context_window"], 200_000);
assert_eq!(parsed["data"]["greeting"]["context_tokens"], 42_000); assert_eq!(parsed["data"]["greeting"]["context_tokens"], 42_000);
assert_eq!(parsed["data"]["status"], "paused"); assert_eq!(parsed["data"]["state"]["state"]["kind"], "busy");
assert_eq!(parsed["data"]["state"]["state"]["state"]["state"], "paused");
} }
#[test] #[test]
fn event_snapshot_in_flight_roundtrip_and_default() { fn event_snapshot_in_flight_roundtrip_and_default() {
let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"status":"running"}}"#; let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"state":{"execution_generation":1,"revision":1,"last_command_id":0,"state":{"kind":"busy","state":{"kind":"run","state":"running"}}}}}"#;
let decoded: Event = serde_json::from_str(inbound).unwrap(); let decoded: Event = serde_json::from_str(inbound).unwrap();
match decoded { match decoded {
Event::Snapshot { in_flight, .. } => assert!(in_flight.is_empty()), Event::Snapshot { in_flight, .. } => assert!(in_flight.is_empty()),
@@ -1946,7 +2208,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Running, state: WorkerStatus::Running.into(),
in_flight: InFlightSnapshot { in_flight: InFlightSnapshot {
blocks: vec![ blocks: vec![
InFlightBlock::Text { InFlightBlock::Text {
@@ -2034,20 +2296,32 @@ mod tests {
} }
#[test] #[test]
fn event_status_format() { fn event_worker_state_format() {
let event = Event::Status { let event = Event::WorkerState {
status: WorkerStatus::Running, snapshot: WorkerStateSnapshot {
execution_generation: 7,
revision: 3,
last_command_id: 9,
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
},
}; };
let json = serde_json::to_string(&event).unwrap(); let json = serde_json::to_string(&event).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["event"], "status"); assert_eq!(parsed["event"], "worker_state");
assert_eq!(parsed["data"]["status"], "running"); assert_eq!(parsed["data"]["snapshot"]["execution_generation"], 7);
assert_eq!(parsed["data"]["snapshot"]["revision"], 3);
assert_eq!(parsed["data"]["snapshot"]["state"]["kind"], "busy");
let decoded: Event = serde_json::from_str(&json).unwrap(); let decoded: Event = serde_json::from_str(&json).unwrap();
assert!(matches!( assert!(matches!(
decoded, decoded,
Event::Status { Event::WorkerState {
status: WorkerStatus::Running snapshot: WorkerStateSnapshot {
execution_generation: 7,
revision: 3,
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
..
}
} }
)); ));
} }
@@ -2088,19 +2362,10 @@ mod tests {
} }
#[test] #[test]
fn event_snapshot_without_status_defaults_to_idle() { fn event_snapshot_without_worker_state_fails_closed() {
let json = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#; let json = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#;
let decoded: Event = serde_json::from_str(json).unwrap(); let error = serde_json::from_str::<Event>(json).unwrap_err();
match decoded { assert!(error.to_string().contains("state"));
Event::Snapshot {
status, greeting, ..
} => {
assert_eq!(status, WorkerStatus::Idle);
assert_eq!(greeting.context_window, 0);
assert_eq!(greeting.context_tokens, 0);
}
other => panic!("expected Snapshot, got {other:?}"),
}
} }
#[test] #[test]
@@ -2513,7 +2778,12 @@ mod tests {
"scope_summary": "scope", "scope_summary": "scope",
"tools": [] "tools": []
}, },
"status": "idle" "state": {
"execution_generation": 1,
"revision": 0,
"last_command_id": 0,
"state": { "kind": "idle" }
}
} }
})) }))
.unwrap(); .unwrap();
+6
View File
@@ -573,6 +573,11 @@ pub struct SubscriptionWorker {
pub resource_key: Option<String>, pub resource_key: Option<String>,
/// Producer-owned monotonic revision for this Worker subject. /// Producer-owned monotonic revision for this Worker subject.
pub subject_revision: u64, pub subject_revision: u64,
/// Latest revisioned foreground state observed from the Worker. This remains
/// absent until an authoritative Worker snapshot/event has been applied.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<crate::WorkerStateSnapshot>,
/// Runtime catalog lifecycle compatibility projection; not foreground-state authority.
pub state: SubscriptionWorkerState, pub state: SubscriptionWorkerState,
#[serde(default)] #[serde(default)]
pub has_running_internal_workers: bool, pub has_running_internal_workers: bool,
@@ -874,6 +879,7 @@ mod tests {
runtime_id: None, runtime_id: None,
resource_key: None, resource_key: None,
subject_revision: 0, subject_revision: 0,
worker_state: None,
state: SubscriptionWorkerState::Idle, state: SubscriptionWorkerState::Idle,
has_running_internal_workers: false, has_running_internal_workers: false,
workspace_id: Some("workspace-1".to_string()), workspace_id: Some("workspace-1".to_string()),
+13 -1
View File
@@ -12,7 +12,10 @@ use crate::{
RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart, RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart,
SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry, SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry,
SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition, SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition,
TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, WorkerStatus, TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerBusyState,
WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope,
WorkerCommandKind, WorkerEvent, WorkerMaintenanceState, WorkerRunState, WorkerState,
WorkerStateSnapshot, WorkerStatus,
subscription::{ subscription::{
EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame, EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame,
SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest, SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest,
@@ -46,6 +49,15 @@ pub fn generated_protocol_types() -> String {
push_decl::<AlertSource>(&cfg, &mut output); push_decl::<AlertSource>(&cfg, &mut output);
push_decl::<CompletionKind>(&cfg, &mut output); push_decl::<CompletionKind>(&cfg, &mut output);
push_decl::<WorkerStatus>(&cfg, &mut output); push_decl::<WorkerStatus>(&cfg, &mut output);
push_decl::<WorkerCommandEnvelope>(&cfg, &mut output);
push_decl::<WorkerCommandKind>(&cfg, &mut output);
push_decl::<WorkerCommandDisposition>(&cfg, &mut output);
push_decl::<WorkerCommandAcknowledgement>(&cfg, &mut output);
push_decl::<WorkerRunState>(&cfg, &mut output);
push_decl::<WorkerMaintenanceState>(&cfg, &mut output);
push_decl::<WorkerBusyState>(&cfg, &mut output);
push_decl::<WorkerState>(&cfg, &mut output);
push_decl::<WorkerStateSnapshot>(&cfg, &mut output);
push_decl::<TurnResult>(&cfg, &mut output); push_decl::<TurnResult>(&cfg, &mut output);
push_decl::<InvokeKind>(&cfg, &mut output); push_decl::<InvokeKind>(&cfg, &mut output);
push_decl::<RunResult>(&cfg, &mut output); push_decl::<RunResult>(&cfg, &mut output);
+10 -2
View File
@@ -318,7 +318,11 @@ impl StandaloneHost {
} }
pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> { pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> {
let _ = self.handle.send(Method::Shutdown).await; let command = protocol::WorkerCommandEnvelope::for_snapshot(
u64::MAX,
&self.handle.shared_state.snapshot(),
);
let _ = self.handle.send(Method::Shutdown { command }).await;
let Some(shutdown) = self.shutdown.take() else { let Some(shutdown) = self.shutdown.take() else {
self.retain_lease(); self.retain_lease();
return Err(StandaloneShutdownError::ConfirmationLost); return Err(StandaloneShutdownError::ConfirmationLost);
@@ -500,7 +504,11 @@ fn active_pointer(
} }
async fn stop_started_worker(started: BootstrappedWorker) { async fn stop_started_worker(started: BootstrappedWorker) {
let _ = started.handle.send(Method::Shutdown).await; let command = protocol::WorkerCommandEnvelope::for_snapshot(
u64::MAX,
&started.handle.shared_state.snapshot(),
);
let _ = started.handle.send(Method::Shutdown { command }).await;
let _ = tokio::time::timeout(Duration::from_secs(2), started.shutdown).await; let _ = tokio::time::timeout(Duration::from_secs(2), started.shutdown).await;
} }
+158 -38
View File
@@ -5,7 +5,7 @@ use std::time::{Duration, Instant};
use protocol::{ use protocol::{
AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, InFlightBlock, AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, InFlightBlock,
InFlightSnapshot, InFlightToolCallState, InternalWorkerRef, InternalWorkerSnapshot, Method, InFlightSnapshot, InFlightToolCallState, InternalWorkerRef, InternalWorkerSnapshot, Method,
RewindTarget, RunResult, Segment, WorkerStatus, RewindTarget, RunResult, Segment, WorkerCommandEnvelope, WorkerStateSnapshot, WorkerStatus,
}; };
use crate::block::{ use crate::block::{
@@ -225,8 +225,10 @@ pub struct WorkerViewTab {
pub struct App { pub struct App {
pub worker_name: String, pub worker_name: String,
pub connected: bool, pub connected: bool,
/// Last controller status reported by the Worker. Drives the status line /// Latest authoritative revisioned live execution state.
/// and Ctrl-key routing; do not infer this solely from replayed history. pub worker_state: WorkerStateSnapshot,
next_command_id: u64,
/// Derived Runtime-catalog compatibility projection used by existing UI.
pub worker_status: WorkerStatus, pub worker_status: WorkerStatus,
/// True while the Worker is in `WorkerStatus::Running`. /// True while the Worker is in `WorkerStatus::Running`.
pub running: bool, pub running: bool,
@@ -337,6 +339,8 @@ impl App {
Self { Self {
worker_name, worker_name,
connected: false, connected: false,
worker_state: WorkerStateSnapshot::initial(1),
next_command_id: 1,
worker_status: WorkerStatus::Idle, worker_status: WorkerStatus::Idle,
running: false, running: false,
paused: false, paused: false,
@@ -745,7 +749,8 @@ impl App {
if self.paused { if self.paused {
self.input_history.cancel_browse(); self.input_history.cancel_browse();
self.input.clear(); self.input.clear();
return Some(Method::Resume); let command = self.next_command_envelope();
return Some(Method::Resume { command });
} }
return None; return None;
} }
@@ -1114,6 +1119,31 @@ impl App {
} }
} }
pub fn next_command_envelope(&mut self) -> WorkerCommandEnvelope {
let command_id = self
.next_command_id
.max(self.worker_state.last_command_id.saturating_add(1));
let command = WorkerCommandEnvelope::for_snapshot(command_id, &self.worker_state);
self.next_command_id = command_id.saturating_add(1);
command
}
fn apply_worker_state_snapshot(&mut self, snapshot: &WorkerStateSnapshot) {
match protocol::apply_worker_state_snapshot(&mut self.worker_state, snapshot) {
Ok(protocol::WorkerStateSnapshotApply::Applied) => {
self.set_worker_status(self.worker_state.catalog_status());
}
Ok(
protocol::WorkerStateSnapshotApply::Duplicate
| protocol::WorkerStateSnapshotApply::Stale,
) => {}
Err(error) => self.handle_error(
ErrorCode::Internal,
format!("worker state stream rejected: {error}"),
),
}
}
pub fn handle_worker_event(&mut self, event: Event) -> Option<Method> { pub fn handle_worker_event(&mut self, event: Event) -> Option<Method> {
if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) { if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) {
return None; return None;
@@ -1150,18 +1180,14 @@ impl App {
self.assistant_streaming = false; self.assistant_streaming = false;
} }
Event::TurnStart { .. } => { Event::TurnStart { .. } => {
self.set_worker_status(WorkerStatus::Running);
self.run_requests += 1; self.run_requests += 1;
self.current_tool = None; self.current_tool = None;
self.latest_llm_wait_event = None; self.latest_llm_wait_event = None;
self.assistant_streaming = false; self.assistant_streaming = false;
} }
Event::InvokeStart { .. } => { Event::InvokeStart { .. } => {}
self.set_worker_status(WorkerStatus::Running);
}
// UI consumers of per-attempt LlmCall semantics remain out of scope; // UI consumers of per-attempt LlmCall semantics remain out of scope;
// the run-level status starts at InvokeStart and TurnStart counts each // authoritative run state comes only from WorkerStateSnapshot.
// LLM request within that run.
Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => { Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => {
self.latest_llm_wait_event = None; self.latest_llm_wait_event = None;
} }
@@ -1368,12 +1394,7 @@ impl App {
output_tokens: self.run_output_tokens, output_tokens: self.run_output_tokens,
}); });
self.pending_submit_rollback = None; self.pending_submit_rollback = None;
self.reset_run_state(match result { self.reset_run_state();
RunResult::Paused => WorkerStatus::Paused,
RunResult::Finished | RunResult::LimitReached | RunResult::RolledBack => {
WorkerStatus::Idle
}
});
} }
} }
Event::CompactStart { .. } => { Event::CompactStart { .. } => {
@@ -1443,7 +1464,7 @@ impl App {
Event::Snapshot { Event::Snapshot {
session, session,
greeting, greeting,
status, state,
in_flight, in_flight,
internal_workers, internal_workers,
} => { } => {
@@ -1451,7 +1472,7 @@ impl App {
self.pending_submissions = session.pending_submissions.clone(); self.pending_submissions = session.pending_submissions.clone();
self.restore_snapshot(&session, greeting, in_flight); self.restore_snapshot(&session, greeting, in_flight);
self.replace_internal_worker_snapshots(internal_workers); self.replace_internal_worker_snapshots(internal_workers);
self.set_worker_status(status); self.apply_worker_state_snapshot(&state);
} }
Event::InternalWorker { Event::InternalWorker {
worker, worker,
@@ -1461,9 +1482,12 @@ impl App {
Event::InternalWorkerRemoved { worker, revision } => { Event::InternalWorkerRemoved { worker, revision } => {
self.remove_internal_worker(worker, revision) self.remove_internal_worker(worker, revision)
} }
Event::Status { status } => { Event::WorkerState { snapshot } => {
self.rewind_refresh_fence = false; self.rewind_refresh_fence = false;
self.set_worker_status(status); self.apply_worker_state_snapshot(&snapshot);
}
Event::CommandAcknowledged { acknowledgement } => {
self.apply_worker_state_snapshot(&acknowledgement.state);
} }
// Command telemetry is an operational Web Console surface. The // Command telemetry is an operational Web Console surface. The
// TUI continues to render the final Bash ToolResult from history. // TUI continues to render the final Bash ToolResult from history.
@@ -1503,7 +1527,7 @@ impl App {
}; };
self.completion = None; self.completion = None;
self.close_rewind_picker(); self.close_rewind_picker();
self.reset_run_state(self.worker_status); self.reset_run_state();
let mut message = if restored_composer { let mut message = if restored_composer {
format!( format!(
"Rewound session: discarded {} log entries; restored selected input to composer.", "Rewound session: discarded {} log entries; restored selected input to composer.",
@@ -1551,8 +1575,7 @@ impl App {
None None
} }
fn reset_run_state(&mut self, status: WorkerStatus) { fn reset_run_state(&mut self) {
self.set_worker_status(status);
self.run_requests = 0; self.run_requests = 0;
self.run_upload_tokens = 0; self.run_upload_tokens = 0;
self.run_output_tokens = 0; self.run_output_tokens = 0;
@@ -1582,7 +1605,7 @@ impl App {
"Rolled back empty assistant turn; no local submitted input was available to restore." "Rolled back empty assistant turn; no local submitted input was available to restore."
.to_owned() .to_owned()
}; };
self.reset_run_state(WorkerStatus::Idle); self.reset_run_state();
self.blocks.push(Block::Alert { self.blocks.push(Block::Alert {
level: AlertLevel::Warn, level: AlertLevel::Warn,
source: AlertSource::Worker, source: AlertSource::Worker,
@@ -2026,12 +2049,18 @@ impl App {
self.input_mode = CommandInputMode::Composer; self.input_mode = CommandInputMode::Composer;
self.command_completion_selected = None; self.command_completion_selected = None;
} }
if let Some(Method::ListRewindTargets) = result.method.as_ref() { let mut method = result.method;
if let Some(Method::Compact { .. }) = method {
method = Some(Method::Compact {
command: self.next_command_envelope(),
});
}
if let Some(Method::ListRewindTargets) = method.as_ref() {
self.completion = None; self.completion = None;
self.rewind_picker = None; self.rewind_picker = None;
self.rewind_request_pending = true; self.rewind_request_pending = true;
} }
result.method method
} }
fn push_command_diagnostic(&mut self, message: impl Into<String>) { fn push_command_diagnostic(&mut self, message: impl Into<String>) {
@@ -2761,8 +2790,8 @@ mod rewind_refresh_tests {
}); });
assert!(!blocks_contain(&app, "stale tail after rewind")); assert!(!blocks_contain(&app, "stale tail after rewind"));
app.handle_worker_event(Event::Status { app.handle_worker_event(Event::WorkerState {
status: WorkerStatus::Idle, snapshot: WorkerStatus::Idle.into(),
}); });
app.handle_worker_event(Event::TextDelta { app.handle_worker_event(Event::TextDelta {
text: "new live tail after status".into(), text: "new live tail after status".into(),
@@ -3478,7 +3507,7 @@ mod completion_flow_tests {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
app.set_worker_status(WorkerStatus::Paused); app.set_worker_status(WorkerStatus::Paused);
assert!(matches!(app.submit_input(), Some(Method::Resume))); assert!(matches!(app.submit_input(), Some(Method::Resume { .. })));
assert_eq!(app.queued_input_count(), 0); assert_eq!(app.queued_input_count(), 0);
} }
@@ -3533,7 +3562,7 @@ mod completion_flow_tests {
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: public_session(vec![session_start_value]), session: public_session(vec![session_start_value]),
status: WorkerStatus::Running, state: test_worker_state(WorkerStatus::Running),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
@@ -3544,6 +3573,90 @@ mod completion_flow_tests {
assert!(matches!(app.blocks.first(), Some(Block::Greeting(_)))); assert!(matches!(app.blocks.first(), Some(Block::Greeting(_))));
} }
#[test]
fn occurrence_events_do_not_infer_foreground_worker_state() {
let mut app = App::new("test".into());
app.handle_worker_event(Event::TurnStart { turn: 1 });
app.handle_worker_event(Event::InvokeStart {
kind: protocol::InvokeKind::UserSend,
});
app.handle_worker_event(Event::RunEnd {
result: RunResult::Paused,
});
assert_eq!(app.worker_state.state, protocol::WorkerState::Idle);
assert_eq!(app.worker_status, WorkerStatus::Idle);
let running = WorkerStateSnapshot {
execution_generation: 1,
revision: 1,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 0,
};
app.handle_worker_event(Event::WorkerState {
snapshot: running.clone(),
});
app.handle_worker_event(Event::RunEnd {
result: RunResult::Finished,
});
assert_eq!(app.worker_state, running);
assert_eq!(app.worker_status, WorkerStatus::Running);
}
#[test]
fn worker_state_events_and_acknowledgements_share_monotonic_application() {
let mut app = App::new("test".into());
let running = WorkerStateSnapshot {
execution_generation: 4,
revision: 3,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
};
app.handle_worker_event(Event::WorkerState {
snapshot: running.clone(),
});
app.handle_worker_event(Event::WorkerState {
snapshot: WorkerStateSnapshot {
revision: 2,
state: protocol::WorkerState::Idle,
..running.clone()
},
});
assert_eq!(app.worker_state, running);
let paused = WorkerStateSnapshot {
revision: 4,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Paused,
)),
last_command_id: 3,
..running.clone()
};
app.handle_worker_event(Event::CommandAcknowledged {
acknowledgement: protocol::WorkerCommandAcknowledgement {
command_id: 3,
command: protocol::WorkerCommandKind::Pause,
disposition: protocol::WorkerCommandDisposition::Accepted,
state: paused.clone(),
},
});
assert_eq!(app.worker_state, paused);
app.handle_worker_event(Event::WorkerState {
snapshot: WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
..paused.clone()
},
});
assert_eq!(app.worker_state, paused);
assert!(app.run_error_messages.iter().any(|message| {
message.contains("conflicting worker state snapshots at generation 4 revision 4")
}));
}
#[test] #[test]
fn snapshot_replaces_live_error_with_one_durable_run_error_block() { fn snapshot_replaces_live_error_with_one_durable_run_error_block() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
@@ -3551,8 +3664,8 @@ mod completion_flow_tests {
code: ErrorCode::ProviderError, code: ErrorCode::ProviderError,
message: "provider unavailable".into(), message: "provider unavailable".into(),
}); });
app.handle_worker_event(Event::Status { app.handle_worker_event(Event::WorkerState {
status: WorkerStatus::Idle, snapshot: WorkerStatus::Idle.into(),
}); });
let live_errors = app let live_errors = app
@@ -3577,7 +3690,7 @@ mod completion_flow_tests {
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: public_session(vec![serde_json::to_value(run_errored).unwrap()]), session: public_session(vec![serde_json::to_value(run_errored).unwrap()]),
status: WorkerStatus::Idle, state: test_worker_state(WorkerStatus::Idle),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
@@ -3641,7 +3754,7 @@ mod completion_flow_tests {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Running, state: test_worker_state(WorkerStatus::Running),
in_flight: InFlightSnapshot { in_flight: InFlightSnapshot {
blocks: vec![ blocks: vec![
InFlightBlock::Thinking { InFlightBlock::Thinking {
@@ -3968,7 +4081,7 @@ mod completion_flow_tests {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Idle, state: test_worker_state(WorkerStatus::Idle),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
@@ -4020,7 +4133,7 @@ mod completion_flow_tests {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Idle, state: test_worker_state(WorkerStatus::Idle),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: vec![InternalWorkerSnapshot { internal_workers: vec![InternalWorkerSnapshot {
worker: InternalWorkerRef { worker: InternalWorkerRef {
@@ -4168,6 +4281,13 @@ mod completion_flow_tests {
.count() .count()
} }
fn test_worker_state(status: WorkerStatus) -> WorkerStateSnapshot {
let mut snapshot = WorkerStateSnapshot::from(status);
snapshot.execution_generation = 1;
snapshot.revision = 1;
snapshot
}
fn test_greeting() -> protocol::Greeting { fn test_greeting() -> protocol::Greeting {
protocol::Greeting { protocol::Greeting {
worker_name: "test".into(), worker_name: "test".into(),
@@ -4194,7 +4314,7 @@ mod completion_flow_tests {
entries: Vec::new(), entries: Vec::new(),
}, },
greeting, greeting,
status: WorkerStatus::Idle, state: test_worker_state(WorkerStatus::Idle),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
@@ -4393,7 +4513,7 @@ mod completion_flow_tests {
app.handle_worker_event(Event::Snapshot { app.handle_worker_event(Event::Snapshot {
greeting: test_greeting(), greeting: test_greeting(),
session: public_session(assistant_item_entries), session: public_session(assistant_item_entries),
status: WorkerStatus::Running, state: test_worker_state(WorkerStatus::Running),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
+31 -3
View File
@@ -321,7 +321,7 @@ fn row_line(
Span::raw(" "), Span::raw(" "),
Span::styled( Span::styled(
pad_column(&worker_state(worker), widths.state), pad_column(&worker_state(worker), widths.state),
state_style(worker.state.as_str()), state_style(worker_state_label(worker)),
), ),
Span::raw(" "), Span::raw(" "),
Span::styled( Span::styled(
@@ -341,8 +341,20 @@ fn worker_name(worker: &BackendWorkerSummary) -> &str {
} }
} }
fn worker_state_label(worker: &BackendWorkerSummary) -> &str {
match worker.worker_state.as_ref().map(|state| &state.state) {
Some(protocol::WorkerState::Idle) => "idle",
Some(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Paused,
))) => "paused",
Some(protocol::WorkerState::Busy(_)) => "running",
None if worker.state == "stopped" => "stopped",
None => "unknown",
}
}
fn worker_state(worker: &BackendWorkerSummary) -> String { fn worker_state(worker: &BackendWorkerSummary) -> String {
format!("[{}]", worker.state) format!("[{}]", worker_state_label(worker))
} }
fn text_width(value: &str) -> usize { fn text_width(value: &str) -> usize {
@@ -413,7 +425,15 @@ mod tests {
identity: "ws".to_string(), identity: "ws".to_string(),
workspace_id: Some("ws".to_string()), workspace_id: Some("ws".to_string()),
}, },
state: "running".to_string(), state: "idle".to_string(),
worker_state: Some(protocol::WorkerStateSnapshot {
execution_generation: 1,
revision: 1,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 0,
}),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: String::new(), retention_state: String::new(),
@@ -450,6 +470,7 @@ mod tests {
worker.display_name = "Coder".to_string(); worker.display_name = "Coder".to_string();
worker.label = "Coder · T-585".to_string(); worker.label = "Coder · T-585".to_string();
worker.state = "stopped".to_string(); worker.state = "stopped".to_string();
worker.worker_state = None;
worker.working_directory = Some( worker.working_directory = Some(
serde_json::from_value(serde_json::json!({ serde_json::from_value(serde_json::json!({
"working_directory_id": "001a06a9f0202000000", "working_directory_id": "001a06a9f0202000000",
@@ -478,12 +499,19 @@ mod tests {
short.label = "Coder".to_string(); short.label = "Coder".to_string();
short.display_name = short.label.clone(); short.display_name = short.label.clone();
short.state = "idle".to_string(); short.state = "idle".to_string();
short.worker_state = Some(protocol::WorkerStateSnapshot {
execution_generation: 1,
revision: 2,
state: protocol::WorkerState::Idle,
last_command_id: 0,
});
let mut long = worker("runtime-a", "worker-b", None); let mut long = worker("runtime-a", "worker-b", None);
long.resource_key = "W-100".to_string(); long.resource_key = "W-100".to_string();
long.label = "Longer worker · T-9".to_string(); long.label = "Longer worker · T-9".to_string();
long.display_name = long.label.clone(); long.display_name = long.label.clone();
long.state = "stopped".to_string(); long.state = "stopped".to_string();
long.worker_state = None;
for worker in [&mut short, &mut long] { for worker in [&mut short, &mut long] {
worker.working_directory = Some( worker.working_directory = Some(
+7 -2
View File
@@ -409,7 +409,12 @@ fn compact_command(invocation: CommandInvocation<'_>) -> CommandExecution {
let _ = invocation.environment; let _ = invocation.environment;
let _ = invocation.args.raw(); let _ = invocation.args.raw();
CommandExecution { CommandExecution {
method: Some(Method::Compact), method: Some(Method::Compact {
command: protocol::WorkerCommandEnvelope::for_snapshot(
0,
&protocol::WorkerStateSnapshot::initial(1),
),
}),
diagnostics: vec![CommandDiagnostic::new("compact requested")], diagnostics: vec![CommandDiagnostic::new("compact requested")],
exit_command_mode: true, exit_command_mode: true,
clear_input: true, clear_input: true,
@@ -483,7 +488,7 @@ mod tests {
fn compact_command_returns_compact_method_not_run() { fn compact_command_returns_compact_method_not_run() {
let registry = CommandRegistry::builtins(); let registry = CommandRegistry::builtins();
let result = registry.dispatch("compact", &env()); let result = registry.dispatch("compact", &env());
assert!(matches!(result.method, Some(Method::Compact))); assert!(matches!(result.method, Some(Method::Compact { .. })));
assert!(result.exit_command_mode); assert!(result.exit_command_mode);
assert!(result.clear_input); assert!(result.clear_input);
assert!(result.diagnostics[0].message.contains("compact requested")); assert!(result.diagnostics[0].message.contains("compact requested"));
+23 -20
View File
@@ -572,7 +572,7 @@ async fn run_e2e_rewind_fixture(
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(), entries: Vec::new(),
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
greeting: Greeting { greeting: Greeting {
worker_name: worker_name.clone(), worker_name: worker_name.clone(),
cwd: workspace_root.display().to_string(), cwd: workspace_root.display().to_string(),
@@ -1438,13 +1438,15 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
WorkerStatus::Running | WorkerStatus::Paused WorkerStatus::Running | WorkerStatus::Paused
) { ) {
app.shutdown_confirm = None; app.shutdown_confirm = None;
return Some(Method::Cancel); let command = app.next_command_envelope();
return Some(Method::Cancel { command });
} }
if let Some(pressed_at) = app.shutdown_confirm if let Some(pressed_at) = app.shutdown_confirm
&& pressed_at.elapsed() < CONFIRM_TIMEOUT && pressed_at.elapsed() < CONFIRM_TIMEOUT
{ {
app.shutdown_confirm = None; app.shutdown_confirm = None;
return Some(Method::Shutdown); let command = app.next_command_envelope();
return Some(Method::Shutdown { command });
} }
app.shutdown_confirm = Some(std::time::Instant::now()); app.shutdown_confirm = Some(std::time::Instant::now());
app.flash_actionbar_notice( app.flash_actionbar_notice(
@@ -1460,7 +1462,8 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
/// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running). /// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running).
fn handle_pause_or_quit(app: &mut App) -> Option<Method> { fn handle_pause_or_quit(app: &mut App) -> Option<Method> {
if app.worker_status == WorkerStatus::Running { if app.worker_status == WorkerStatus::Running {
return Some(Method::Pause); let command = app.next_command_envelope();
return Some(Method::Pause { command });
} }
if let Some(t) = app.quit_confirm if let Some(t) = app.quit_confirm
&& t.elapsed() < CONFIRM_TIMEOUT && t.elapsed() < CONFIRM_TIMEOUT
@@ -2090,7 +2093,7 @@ mod tests {
&mut app, &mut app,
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL), KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL),
), ),
Some(Method::Pause) Some(Method::Pause { .. })
)); ));
assert_eq!(app.queued_input_count(), 1); assert_eq!(app.queued_input_count(), 1);
@@ -2100,7 +2103,7 @@ mod tests {
&mut app, &mut app,
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
), ),
Some(Method::Cancel) Some(Method::Cancel { .. })
)); ));
assert_eq!(app.queued_input_count(), 1); assert_eq!(app.queued_input_count(), 1);
} }
@@ -2114,7 +2117,7 @@ mod tests {
&mut app, &mut app,
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
); );
assert!(matches!(cancel, Some(Method::Cancel))); assert!(matches!(cancel, Some(Method::Cancel { .. })));
} }
#[test] #[test]
@@ -2136,7 +2139,7 @@ mod tests {
assert!(matches!( assert!(matches!(
handle_key(&mut app, ctrl_x()), handle_key(&mut app, ctrl_x()),
Some(Method::Shutdown) Some(Method::Shutdown { .. })
)); ));
assert!(app.shutdown_confirm.is_none()); assert!(app.shutdown_confirm.is_none());
} }
@@ -2466,7 +2469,7 @@ mod tests {
} }
let method = handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); let method = handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
assert!(matches!(method, Some(protocol::Method::Compact))); assert!(matches!(method, Some(protocol::Method::Compact { .. })));
assert!(!app.is_command_mode()); assert!(!app.is_command_mode());
assert_eq!(input_text(&app), ""); assert_eq!(input_text(&app), "");
assert_eq!(app.queued_input_count(), 0); assert_eq!(app.queued_input_count(), 0);
@@ -2573,7 +2576,7 @@ mod tests {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![], entries: vec![],
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
@@ -2606,7 +2609,7 @@ mod tests {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![], entries: vec![],
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}); });
@@ -2743,8 +2746,8 @@ mod tests {
kind: protocol::InternalWorkerKind::SubWorker, kind: protocol::InternalWorkerKind::SubWorker,
}, },
revision: 1, revision: 1,
event: Box::new(Event::Status { event: Box::new(Event::WorkerState {
status: WorkerStatus::Running, snapshot: WorkerStatus::Running.into(),
}), }),
}); });
enter_command_mode(&mut app); enter_command_mode(&mut app);
@@ -2859,8 +2862,8 @@ mod tests {
kind: protocol::InternalWorkerKind::SubWorker, kind: protocol::InternalWorkerKind::SubWorker,
}, },
revision: 1, revision: 1,
event: Box::new(Event::Status { event: Box::new(Event::WorkerState {
status: WorkerStatus::Running, snapshot: WorkerStatus::Running.into(),
}), }),
}); });
@@ -2885,8 +2888,8 @@ mod tests {
kind: protocol::InternalWorkerKind::SubWorker, kind: protocol::InternalWorkerKind::SubWorker,
}, },
revision: 1, revision: 1,
event: Box::new(Event::Status { event: Box::new(Event::WorkerState {
status: WorkerStatus::Running, snapshot: WorkerStatus::Running.into(),
}), }),
}); });
handle_key(&mut app, key(KeyCode::Tab)); handle_key(&mut app, key(KeyCode::Tab));
@@ -2902,7 +2905,7 @@ mod tests {
); );
assert!(first.is_none()); assert!(first.is_none());
assert!(matches!(second, Some(Method::Shutdown))); assert!(matches!(second, Some(Method::Shutdown { .. })));
assert_eq!(app.worker_status, WorkerStatus::Idle); assert_eq!(app.worker_status, WorkerStatus::Idle);
} }
@@ -2924,8 +2927,8 @@ mod tests {
kind: protocol::InternalWorkerKind::SubWorker, kind: protocol::InternalWorkerKind::SubWorker,
}, },
revision: 1, revision: 1,
event: Box::new(Event::Status { event: Box::new(Event::WorkerState {
status: WorkerStatus::Running, snapshot: WorkerStatus::Running.into(),
}), }),
}); });
+6
View File
@@ -307,6 +307,8 @@ pub struct WorkerSummary {
pub worker_id: WorkerId, pub worker_id: WorkerId,
pub status: WorkerStatus, pub status: WorkerStatus,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace_id: Option<String>, pub workspace_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>, pub working_directory: Option<WorkingDirectoryStatus>,
@@ -325,6 +327,8 @@ pub struct WorkerDetail {
pub worker_id: WorkerId, pub worker_id: WorkerId,
pub status: WorkerStatus, pub status: WorkerStatus,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace_id: Option<String>, pub workspace_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>, pub working_directory: Option<WorkingDirectoryStatus>,
@@ -341,6 +345,8 @@ pub struct WorkerDetail {
pub struct WorkerLifecycleAck { pub struct WorkerLifecycleAck {
pub worker_ref: WorkerRef, pub worker_ref: WorkerRef,
pub status: WorkerStatus, pub status: WorkerStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
} }
#[cfg(test)] #[cfg(test)]
+14 -28
View File
@@ -15,18 +15,6 @@ use std::fmt;
use std::sync::Arc; use std::sync::Arc;
use workdir::WorkdirSessionHandle; use workdir::WorkdirSessionHandle;
/// Current execution-side run state for a Worker.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerExecutionRunState {
#[default]
Stopped,
Idle,
Busy,
Rejected,
Errored,
}
/// Execution operation that produced a result. /// Execution operation that produced a result.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
@@ -55,7 +43,8 @@ pub struct WorkerSubmissionAck {
pub struct WorkerExecutionResult { pub struct WorkerExecutionResult {
pub operation: WorkerExecutionOperation, pub operation: WorkerExecutionOperation,
pub outcome: WorkerExecutionOutcome, pub outcome: WorkerExecutionOutcome,
pub run_state: WorkerExecutionRunState, #[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub message: Option<String>, pub message: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -74,22 +63,23 @@ pub enum WorkerExecutionOutcome {
} }
impl WorkerExecutionResult { impl WorkerExecutionResult {
pub fn accepted( pub fn accepted(operation: WorkerExecutionOperation) -> Self {
operation: WorkerExecutionOperation,
run_state: WorkerExecutionRunState,
) -> Self {
Self { Self {
operation, operation,
outcome: WorkerExecutionOutcome::Accepted, outcome: WorkerExecutionOutcome::Accepted,
run_state, worker_state: None,
message: None, message: None,
submission: None, submission: None,
} }
} }
pub fn with_worker_state(mut self, worker_state: protocol::WorkerStateSnapshot) -> Self {
self.worker_state = Some(worker_state);
self
}
pub fn accepted_submission( pub fn accepted_submission(
operation: WorkerExecutionOperation, operation: WorkerExecutionOperation,
run_state: WorkerExecutionRunState,
submission_request_id: impl Into<String>, submission_request_id: impl Into<String>,
submission_id: impl Into<String>, submission_id: impl Into<String>,
disposition: protocol::SubmissionDisposition, disposition: protocol::SubmissionDisposition,
@@ -97,7 +87,7 @@ impl WorkerExecutionResult {
Self { Self {
operation, operation,
outcome: WorkerExecutionOutcome::Accepted, outcome: WorkerExecutionOutcome::Accepted,
run_state, worker_state: None,
message: None, message: None,
submission: Some(WorkerSubmissionAck { submission: Some(WorkerSubmissionAck {
submission_request_id: submission_request_id.into(), submission_request_id: submission_request_id.into(),
@@ -111,7 +101,7 @@ impl WorkerExecutionResult {
Self { Self {
operation, operation,
outcome: WorkerExecutionOutcome::Busy, outcome: WorkerExecutionOutcome::Busy,
run_state: WorkerExecutionRunState::Busy, worker_state: None,
message: Some(message.into()), message: Some(message.into()),
submission: None, submission: None,
} }
@@ -121,7 +111,7 @@ impl WorkerExecutionResult {
Self { Self {
operation, operation,
outcome: WorkerExecutionOutcome::Rejected, outcome: WorkerExecutionOutcome::Rejected,
run_state: WorkerExecutionRunState::Stopped, worker_state: None,
message: Some(message.into()), message: Some(message.into()),
submission: None, submission: None,
} }
@@ -131,7 +121,7 @@ impl WorkerExecutionResult {
Self { Self {
operation, operation,
outcome: WorkerExecutionOutcome::Errored, outcome: WorkerExecutionOutcome::Errored,
run_state: WorkerExecutionRunState::Errored, worker_state: None,
message: Some(message.into()), message: Some(message.into()),
submission: None, submission: None,
} }
@@ -141,7 +131,7 @@ impl WorkerExecutionResult {
Self { Self {
operation, operation,
outcome: WorkerExecutionOutcome::Unsupported, outcome: WorkerExecutionOutcome::Unsupported,
run_state: WorkerExecutionRunState::Stopped, worker_state: None,
message: Some(message.into()), message: Some(message.into()),
submission: None, submission: None,
} }
@@ -280,7 +270,6 @@ pub struct WorkerExecutionRestoreRequest {
pub enum WorkerExecutionSpawnResult { pub enum WorkerExecutionSpawnResult {
Connected { Connected {
handle: WorkerExecutionHandle, handle: WorkerExecutionHandle,
run_state: WorkerExecutionRunState,
working_directory: Option<WorkingDirectoryStatus>, working_directory: Option<WorkingDirectoryStatus>,
}, },
Rejected(WorkerExecutionResult), Rejected(WorkerExecutionResult),
@@ -290,12 +279,10 @@ pub enum WorkerExecutionSpawnResult {
impl WorkerExecutionSpawnResult { impl WorkerExecutionSpawnResult {
pub fn connected( pub fn connected(
handle: WorkerExecutionHandle, handle: WorkerExecutionHandle,
run_state: WorkerExecutionRunState,
working_directory: Option<WorkingDirectoryStatus>, working_directory: Option<WorkingDirectoryStatus>,
) -> Self { ) -> Self {
Self::Connected { Self::Connected {
handle, handle,
run_state,
working_directory, working_directory,
} }
} }
@@ -623,7 +610,6 @@ mod tests {
fn submission_ack_survives_json_round_trip() { fn submission_ack_survives_json_round_trip() {
let result = WorkerExecutionResult::accepted_submission( let result = WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
"request-1", "request-1",
"submission-1", "submission-1",
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
+46 -30
View File
@@ -2205,8 +2205,8 @@ mod tests {
}; };
use crate::execution::{ use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest,
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, WorkerExecutionSpawnResult,
}; };
use crate::management::RuntimeOptions; use crate::management::RuntimeOptions;
use axum::body::to_bytes; use axum::body::to_bytes;
@@ -2895,7 +2895,6 @@ mod tests {
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request working_directory: request
.working_directory .working_directory
.as_ref() .as_ref()
@@ -2909,7 +2908,6 @@ mod tests {
) -> WorkerExecutionSpawnResult { ) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request.previous_working_directory, working_directory: request.previous_working_directory,
} }
} }
@@ -2922,24 +2920,17 @@ mod tests {
if let Some(submission_id) = input.submission_request_id { if let Some(submission_id) = input.submission_request_id {
WorkerExecutionResult::accepted_submission( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
submission_id.clone(), submission_id.clone(),
submission_id, submission_id,
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
) )
} else { } else {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
)
} }
} }
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
WorkerExecutionOperation::Stop,
WorkerExecutionRunState::Stopped,
)
} }
} }
@@ -3211,8 +3202,7 @@ mod ws_tests {
}; };
use crate::execution::{ use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
WorkerExecutionSpawnResult,
}; };
use crate::management::RuntimeOptions; use crate::management::RuntimeOptions;
use futures::{SinkExt, StreamExt}; use futures::{SinkExt, StreamExt};
@@ -3232,7 +3222,6 @@ mod ws_tests {
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request working_directory: request
.working_directory .working_directory
.as_ref() .as_ref()
@@ -3248,28 +3237,46 @@ mod ws_tests {
if let Some(submission_id) = input.submission_request_id { if let Some(submission_id) = input.submission_request_id {
WorkerExecutionResult::accepted_submission( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
submission_id.clone(), submission_id.clone(),
submission_id, submission_id,
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
) )
} else { } else {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
)
} }
} }
fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option<protocol::Event> {
Some(protocol::Event::Snapshot {
session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(),
},
greeting: protocol::Greeting {
worker_name: handle.worker_ref().worker_id.to_string(),
cwd: String::new(),
provider: "ws-test".to_string(),
model: "ws-test".to_string(),
scope_summary: "WebSocket test execution snapshot".to_string(),
tools: Vec::new(),
context_window: 0,
context_tokens: 0,
},
state: protocol::WorkerStateSnapshot::initial(1),
in_flight: protocol::InFlightSnapshot {
blocks: Vec::new(),
commands: Vec::new(),
},
internal_workers: Vec::new(),
})
}
fn dispatch_method( fn dispatch_method(
&self, &self,
_handle: &WorkerExecutionHandle, _handle: &WorkerExecutionHandle,
_method: protocol::Method, _method: protocol::Method,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::ProtocolMethod)
WorkerExecutionOperation::ProtocolMethod,
WorkerExecutionRunState::Idle,
)
} }
} }
@@ -3480,16 +3487,16 @@ mod ws_tests {
runtime runtime
.observe_worker_event( .observe_worker_event(
&other.worker_ref, &other.worker_ref,
protocol::Event::Status { protocol::Event::WorkerState {
status: protocol::WorkerStatus::Running, snapshot: protocol::WorkerStatus::Running.into(),
}, },
) )
.unwrap(); .unwrap();
runtime runtime
.observe_worker_event( .observe_worker_event(
&worker_ref, &worker_ref,
protocol::Event::Status { protocol::Event::WorkerState {
status: protocol::WorkerStatus::Running, snapshot: protocol::WorkerStatus::Running.into(),
}, },
) )
.unwrap(); .unwrap();
@@ -3504,7 +3511,16 @@ mod ws_tests {
.. ..
}) if delivered_subscription_id == subscription_id }) if delivered_subscription_id == subscription_id
&& worker.worker_id.as_str() == worker_ref.worker_id.to_string() && worker.worker_id.as_str() == worker_ref.worker_id.to_string()
&& worker.state == protocol::subscription::SubscriptionWorkerState::Running && worker.state == protocol::subscription::SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running
)),
..
})
)
)); ));
let unsubscribe_request_id = let unsubscribe_request_id =
+229 -172
View File
@@ -13,8 +13,8 @@ use crate::error::RuntimeError;
use crate::execution::WorkerExecutionRestoreRequest; use crate::execution::WorkerExecutionRestoreRequest;
use crate::execution::{ use crate::execution::{
WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle, WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle,
WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest,
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, WorkerExecutionSpawnResult,
}; };
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
use crate::fs_store::{ use crate::fs_store::{
@@ -700,6 +700,7 @@ impl Runtime {
worker_ref: worker_ref.clone(), worker_ref: worker_ref.clone(),
worker_id: worker_id.clone(), worker_id: worker_id.clone(),
status: WorkerStatus::Stopped, status: WorkerStatus::Stopped,
worker_state: None,
workspace_id: scope.map(|scope| scope.workspace_id.clone()), workspace_id: scope.map(|scope| scope.workspace_id.clone()),
request: durable_request, request: durable_request,
run_generation: 1, run_generation: 1,
@@ -725,12 +726,11 @@ impl Runtime {
}; };
let spawn_result = backend.spawn_worker(spawn_request); let spawn_result = backend.spawn_worker(spawn_request);
let (handle, run_state, working_directory) = match spawn_result { let (handle, working_directory) = match spawn_result {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle, handle,
run_state,
working_directory, working_directory,
} => (handle, run_state, working_directory), } => (handle, working_directory),
WorkerExecutionSpawnResult::Rejected(result) WorkerExecutionSpawnResult::Rejected(result)
| WorkerExecutionSpawnResult::Errored(result) => { | WorkerExecutionSpawnResult::Errored(result) => {
self.rollback_failed_create(&worker_ref)?; self.rollback_failed_create(&worker_ref)?;
@@ -785,11 +785,9 @@ impl Runtime {
result, result,
}); });
} }
let initial_run_state = dispatch_result.run_state;
let detail = self.commit_created_worker( let detail = self.commit_created_worker(
&worker_ref, &worker_ref,
handle, handle,
initial_run_state,
working_directory, working_directory,
dispatch_result, dispatch_result,
)?; )?;
@@ -799,9 +797,8 @@ impl Runtime {
self.commit_created_worker( self.commit_created_worker(
&worker_ref, &worker_ref,
handle, handle,
run_state,
working_directory, working_directory,
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn, run_state), WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
) )
} }
} }
@@ -1086,13 +1083,12 @@ impl Runtime {
match backend.restore_worker(request) { match backend.restore_worker(request) {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle, handle,
run_state,
working_directory, working_directory,
} => { } => {
self.commit_restored_worker_execution( self.commit_restored_worker_execution(
worker_ref, worker_ref,
handle, handle,
run_state, WorkerStatus::Idle,
working_directory, working_directory,
)?; )?;
self.worker_detail(worker_ref) self.worker_detail(worker_ref)
@@ -1222,7 +1218,9 @@ impl Runtime {
let mut state = self.lock()?; let mut state = self.lock()?;
state.ensure_running()?; state.ensure_running()?;
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
worker.status = worker_status_from_run_state(dispatch_result.run_state); if let Some(snapshot) = dispatch_result.worker_state.as_ref() {
let _ = worker.apply_worker_state(snapshot);
}
let status = worker.status; let status = worker.status;
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
if let Some(payload) = input_protocol_event(&input) { if let Some(payload) = input_protocol_event(&input) {
@@ -1431,7 +1429,7 @@ impl Runtime {
let entries = self.worker_completions(worker_ref, kind, &prefix)?; let entries = self.worker_completions(worker_ref, kind, &prefix)?;
return Ok(vec![Event::Completions { kind, entries }]); return Ok(vec![Event::Completions { kind, entries }]);
} }
if matches!(&method, Method::Shutdown) { if matches!(&method, Method::Shutdown { .. }) {
self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?; self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?;
return Ok(Vec::new()); return Ok(Vec::new());
} }
@@ -1481,16 +1479,19 @@ impl Runtime {
&self, &self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
handle: WorkerExecutionHandle, handle: WorkerExecutionHandle,
run_state: WorkerExecutionRunState,
working_directory: Option<CatalogWorkingDirectoryStatus>, working_directory: Option<CatalogWorkingDirectoryStatus>,
_result: WorkerExecutionResult, result: WorkerExecutionResult,
) -> Result<WorkerDetail, RuntimeError> { ) -> Result<WorkerDetail, RuntimeError> {
let mut state = self.lock()?; let mut state = self.lock()?;
let detail = { let detail = {
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
worker.execution_handle = Some(handle); worker.execution_handle = Some(handle);
worker.execution_bound = true; worker.execution_bound = true;
worker.status = worker_status_from_run_state(run_state); worker.status = WorkerStatus::Idle;
worker.worker_state = None;
if let Some(snapshot) = result.worker_state.as_ref() {
let _ = worker.apply_worker_state(snapshot);
}
worker.restore_intent = restore_intent_for_status(worker.status); worker.restore_intent = restore_intent_for_status(worker.status);
worker.working_directory = working_directory; worker.working_directory = working_directory;
worker.detail() worker.detail()
@@ -1518,16 +1519,26 @@ impl Runtime {
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
result: WorkerExecutionResult, result: WorkerExecutionResult,
) -> Result<(), RuntimeError> { ) -> Result<(), RuntimeError> {
let mut state = self.lock()?; // Accepted dispatch without a state snapshot is transport evidence only;
if result.is_accepted() { // the revisioned protocol stream remains live authority. Test/detached
let status = worker_status_from_run_state(result.run_state); // backends may return an exact full snapshot as their acknowledgement.
let worker = state.worker_mut(worker_ref)?; if !result.is_accepted() {
worker.status = status; return Ok(());
worker.restore_intent = restore_intent_for_status(status);
state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?;
} }
let Some(snapshot) = result.worker_state else {
return Ok(());
};
let mut state = self.lock()?;
let worker = state.worker_mut(worker_ref)?;
let applied = worker
.apply_worker_state(&snapshot)
.is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied));
if !applied {
return Ok(());
}
state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?;
Ok(()) Ok(())
} }
@@ -1615,20 +1626,26 @@ impl Runtime {
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
reason: Option<String>, reason: Option<String>,
) -> Result<WorkerLifecycleAck, RuntimeError> { ) -> Result<WorkerLifecycleAck, RuntimeError> {
let current = { {
let state = self.lock()?; let state = self.lock()?;
state.ensure_running()?; state.ensure_running()?;
state.worker(worker_ref)?.status if state.worker(worker_ref)?.status == WorkerStatus::Stopped {
}; return Ok(WorkerLifecycleAck {
if matches!(current, WorkerStatus::Idle | WorkerStatus::Stopped) { worker_ref: worker_ref.clone(),
return Ok(WorkerLifecycleAck { status: WorkerStatus::Stopped,
worker_ref: worker_ref.clone(), worker_state: None,
status: current, });
}); }
} }
self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?; self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?;
let _ = reason; let _ = reason;
self.transition_worker_preserving_execution(worker_ref, WorkerStatus::Idle) let state = self.lock()?;
let worker = state.worker(worker_ref)?;
Ok(WorkerLifecycleAck {
worker_ref: worker_ref.clone(),
status: worker.status,
worker_state: worker.worker_state.clone(),
})
} }
/// Delete a non-running Worker through a workspace-scoped Runtime authorization context. /// Delete a non-running Worker through a workspace-scoped Runtime authorization context.
@@ -1715,27 +1732,9 @@ impl Runtime {
return Ok(snapshot); return Ok(snapshot);
} }
} }
Ok(protocol::Event::Snapshot { Err(RuntimeError::WorkerExecutionUnavailable {
session: protocol::SessionSnapshot { worker_id: worker_ref.worker_id,
pending_submissions: protocol::PendingSubmissionsSnapshot::default(), message: "authoritative Worker snapshot is unavailable".to_string(),
entries: Vec::new(),
},
greeting: protocol::Greeting {
worker_name: worker_ref.worker_id.to_string(),
cwd: String::new(),
provider: "worker-runtime".to_string(),
model: "worker-runtime".to_string(),
scope_summary: "runtime worker observation".to_string(),
tools: Vec::new(),
context_window: 0,
context_tokens: 0,
},
status: protocol::WorkerStatus::Idle,
in_flight: protocol::InFlightSnapshot {
blocks: Vec::new(),
commands: Vec::new(),
},
internal_workers: Vec::new(),
}) })
} }
@@ -1774,12 +1773,13 @@ impl Runtime {
) -> Result<WorkerObservationEvent, RuntimeError> { ) -> Result<WorkerObservationEvent, RuntimeError> {
let mut state = self.lock()?; let mut state = self.lock()?;
state.ensure_worker_ref(worker_ref)?; state.ensure_worker_ref(worker_ref)?;
let status_changed = state.project_protocol_event_to_status(worker_ref, &payload); let worker_state_changed =
state.project_protocol_event_to_worker_state(worker_ref, &payload);
let activity_changed = state.project_internal_worker_activity(worker_ref, &payload); let activity_changed = state.project_internal_worker_activity(worker_ref, &payload);
if status_changed || activity_changed { if worker_state_changed || activity_changed {
state.publish_worker_upsert(worker_ref.worker_id)?; state.publish_worker_upsert(worker_ref.worker_id)?;
} }
if status_changed { if worker_state_changed {
state.persist_runtime_snapshot()?; state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?; state.persist_worker(&worker_ref.worker_id)?;
} }
@@ -1815,26 +1815,6 @@ impl Runtime {
Ok(()) Ok(())
} }
fn transition_worker_preserving_execution(
&self,
worker_ref: &WorkerRef,
status: WorkerStatus,
) -> Result<WorkerLifecycleAck, RuntimeError> {
let mut state = self.lock()?;
state.ensure_running()?;
let worker = state.worker_mut(worker_ref)?;
worker.status = status;
worker.restore_intent = restore_intent_for_status(status);
let status = worker.status;
state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?;
Ok(WorkerLifecycleAck {
worker_ref: worker_ref.clone(),
status,
})
}
fn transition_worker( fn transition_worker(
&self, &self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
@@ -1846,6 +1826,7 @@ impl Runtime {
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
worker.status = status; worker.status = status;
worker.worker_state = None;
worker.restore_intent = restore_intent_for_status(status); worker.restore_intent = restore_intent_for_status(status);
worker.execution_handle = None; worker.execution_handle = None;
worker.internal_workers.clear(); worker.internal_workers.clear();
@@ -1856,6 +1837,7 @@ impl Runtime {
Ok(WorkerLifecycleAck { Ok(WorkerLifecycleAck {
worker_ref: worker_ref.clone(), worker_ref: worker_ref.clone(),
status, status,
worker_state: None,
}) })
} }
@@ -1968,12 +1950,11 @@ impl Runtime {
match backend.restore_worker(request) { match backend.restore_worker(request) {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle, handle,
run_state,
working_directory, working_directory,
} => self.commit_restored_worker_execution( } => self.commit_restored_worker_execution(
&candidate.worker_ref, &candidate.worker_ref,
handle, handle,
run_state, WorkerStatus::Idle,
working_directory, working_directory,
)?, )?,
WorkerExecutionSpawnResult::Rejected(result) WorkerExecutionSpawnResult::Rejected(result)
@@ -1990,7 +1971,7 @@ impl Runtime {
&self, &self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
handle: WorkerExecutionHandle, handle: WorkerExecutionHandle,
run_state: WorkerExecutionRunState, status: WorkerStatus,
working_directory: Option<CatalogWorkingDirectoryStatus>, working_directory: Option<CatalogWorkingDirectoryStatus>,
) -> Result<(), RuntimeError> { ) -> Result<(), RuntimeError> {
let mut state = self.lock()?; let mut state = self.lock()?;
@@ -1999,7 +1980,7 @@ impl Runtime {
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
worker.execution_handle = Some(handle); worker.execution_handle = Some(handle);
worker.execution_bound = true; worker.execution_bound = true;
worker.status = worker_status_from_run_state(run_state); worker.status = status;
worker.restore_intent = restore_intent_for_status(worker.status); worker.restore_intent = restore_intent_for_status(worker.status);
worker.working_directory = working_directory; worker.working_directory = working_directory;
} }
@@ -2281,6 +2262,7 @@ impl RuntimeState {
worker_ref: worker.worker_ref, worker_ref: worker.worker_ref,
worker_id: worker.worker_id, worker_id: worker.worker_id,
status: worker.status, status: worker.status,
worker_state: None,
workspace_id: worker.workspace_id, workspace_id: worker.workspace_id,
request: worker.request, request: worker.request,
run_generation, run_generation,
@@ -2610,6 +2592,7 @@ impl RuntimeState {
.get(&worker.worker_id) .get(&worker.worker_id)
.copied() .copied()
.unwrap_or(0), .unwrap_or(0),
worker_state: worker.worker_state.clone(),
state: subscription_worker_state(worker.status), state: subscription_worker_state(worker.status),
has_running_internal_workers: worker has_running_internal_workers: worker
.internal_workers .internal_workers
@@ -2867,7 +2850,7 @@ impl RuntimeState {
) { ) {
match event { match event {
protocol::Event::Snapshot { protocol::Event::Snapshot {
status, state,
internal_workers, internal_workers,
.. ..
} => { } => {
@@ -2875,7 +2858,7 @@ impl RuntimeState {
statuses.insert( statuses.insert(
worker.session_id.clone(), worker.session_id.clone(),
InternalWorkerActivity { InternalWorkerActivity {
status: *status, status: state.catalog_status(),
parent_session_id: worker.parent_session_id.clone(), parent_session_id: worker.parent_session_id.clone(),
}, },
); );
@@ -2888,26 +2871,17 @@ impl RuntimeState {
event, event,
.. ..
} => Self::project_internal_worker_event(statuses, nested_worker, event), } => Self::project_internal_worker_event(statuses, nested_worker, event),
protocol::Event::Status { status } => { protocol::Event::WorkerState { snapshot }
statuses.insert( | protocol::Event::CommandAcknowledged {
worker.session_id.clone(), acknowledgement:
InternalWorkerActivity { protocol::WorkerCommandAcknowledgement {
status: *status, state: snapshot, ..
parent_session_id: worker.parent_session_id.clone(),
}, },
); } => {
}
protocol::Event::RunEnd { result } => {
let status = match result {
protocol::RunResult::Paused => protocol::WorkerStatus::Paused,
protocol::RunResult::Finished
| protocol::RunResult::LimitReached
| protocol::RunResult::RolledBack => protocol::WorkerStatus::Idle,
};
statuses.insert( statuses.insert(
worker.session_id.clone(), worker.session_id.clone(),
InternalWorkerActivity { InternalWorkerActivity {
status, status: snapshot.catalog_status(),
parent_session_id: worker.parent_session_id.clone(), parent_session_id: worker.parent_session_id.clone(),
}, },
); );
@@ -2954,7 +2928,7 @@ impl RuntimeState {
Self::update_internal_worker_activity(&mut worker.internal_workers, event) Self::update_internal_worker_activity(&mut worker.internal_workers, event)
} }
fn project_protocol_event_to_status( fn project_protocol_event_to_worker_state(
&mut self, &mut self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
event: &protocol::Event, event: &protocol::Event,
@@ -2962,38 +2936,26 @@ impl RuntimeState {
let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else { let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else {
return false; return false;
}; };
let next_status = match event { let incoming = match event {
protocol::Event::Status { protocol::Event::WorkerState { snapshot }
status: protocol::WorkerStatus::Running, | protocol::Event::Snapshot {
} => Some(WorkerStatus::Running), state: snapshot, ..
protocol::Event::Status { }
status: protocol::WorkerStatus::Idle, | protocol::Event::CommandAcknowledged {
} => Some(WorkerStatus::Idle), acknowledgement:
protocol::Event::Status { protocol::WorkerCommandAcknowledgement {
status: protocol::WorkerStatus::Paused, state: snapshot, ..
} => Some(WorkerStatus::Paused), },
protocol::Event::Snapshot { status, .. } => match status { } => snapshot,
protocol::WorkerStatus::Running => Some(WorkerStatus::Running), _ => return false,
protocol::WorkerStatus::Idle => Some(WorkerStatus::Idle),
protocol::WorkerStatus::Paused => Some(WorkerStatus::Paused),
protocol::WorkerStatus::Stopped => Some(WorkerStatus::Stopped),
},
protocol::Event::RunEnd { result } => match result {
protocol::RunResult::Finished | protocol::RunResult::RolledBack => {
Some(WorkerStatus::Idle)
}
protocol::RunResult::Paused => Some(WorkerStatus::Paused),
protocol::RunResult::LimitReached => Some(WorkerStatus::Idle),
},
_ => None,
}; };
if let Some(next_status) = next_status { match worker.apply_worker_state(incoming) {
let changed = worker.status != next_status; Ok(protocol::WorkerStateSnapshotApply::Applied) => true,
worker.status = next_status; Ok(
worker.restore_intent = restore_intent_for_status(next_status); protocol::WorkerStateSnapshotApply::Duplicate
changed | protocol::WorkerStateSnapshotApply::Stale,
} else { )
false | Err(_) => false,
} }
} }
} }
@@ -3009,6 +2971,7 @@ struct WorkerRecord {
worker_ref: WorkerRef, worker_ref: WorkerRef,
worker_id: WorkerId, worker_id: WorkerId,
status: WorkerStatus, status: WorkerStatus,
worker_state: Option<protocol::WorkerStateSnapshot>,
workspace_id: Option<String>, workspace_id: Option<String>,
request: CreateWorkerRequest, request: CreateWorkerRequest,
run_generation: u64, run_generation: u64,
@@ -3020,6 +2983,19 @@ struct WorkerRecord {
} }
impl WorkerRecord { impl WorkerRecord {
fn apply_worker_state(
&mut self,
incoming: &protocol::WorkerStateSnapshot,
) -> Result<protocol::WorkerStateSnapshotApply, protocol::WorkerStateSnapshotConflict> {
match self.worker_state.as_mut() {
Some(current) => protocol::apply_worker_state_snapshot(current, incoming),
None => {
self.worker_state = Some(incoming.clone());
Ok(protocol::WorkerStateSnapshotApply::Applied)
}
}
}
fn belongs_to_workspace(&self, workspace_id: &str) -> bool { fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
self.workspace_id.as_deref() == Some(workspace_id) self.workspace_id.as_deref() == Some(workspace_id)
} }
@@ -3029,6 +3005,7 @@ impl WorkerRecord {
worker_ref: self.worker_ref.clone(), worker_ref: self.worker_ref.clone(),
worker_id: self.worker_id, worker_id: self.worker_id,
status: self.status, status: self.status,
worker_state: self.worker_state.clone(),
workspace_id: self.workspace_id.clone(), workspace_id: self.workspace_id.clone(),
working_directory: self.working_directory.clone(), working_directory: self.working_directory.clone(),
profile: self.request.profile.clone(), profile: self.request.profile.clone(),
@@ -3043,6 +3020,7 @@ impl WorkerRecord {
worker_ref: self.worker_ref.clone(), worker_ref: self.worker_ref.clone(),
worker_id: self.worker_id, worker_id: self.worker_id,
status: self.status, status: self.status,
worker_state: self.worker_state.clone(),
workspace_id: self.workspace_id.clone(), workspace_id: self.workspace_id.clone(),
working_directory: self.working_directory.clone(), working_directory: self.working_directory.clone(),
profile: self.request.profile.clone(), profile: self.request.profile.clone(),
@@ -3081,16 +3059,6 @@ fn restore_intent_for_status(status: WorkerStatus) -> WorkerRestoreIntent {
} }
} }
fn worker_status_from_run_state(run_state: WorkerExecutionRunState) -> WorkerStatus {
match run_state {
WorkerExecutionRunState::Idle => WorkerStatus::Idle,
WorkerExecutionRunState::Busy => WorkerStatus::Running,
WorkerExecutionRunState::Stopped
| WorkerExecutionRunState::Rejected
| WorkerExecutionRunState::Errored => WorkerStatus::Stopped,
}
}
fn repository_resource_error(error: BackendResourceError) -> RuntimeError { fn repository_resource_error(error: BackendResourceError) -> RuntimeError {
let (code, message) = match error { let (code, message) = match error {
BackendResourceError::Expired => ( BackendResourceError::Expired => (
@@ -3304,7 +3272,7 @@ mod tests {
}; };
use crate::execution::{ use crate::execution::{
WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle, WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle,
WorkerExecutionRestoreRequest, WorkerExecutionRunState, WorkerExecutionRestoreRequest,
}; };
use crate::working_directory::WorkingDirectoryDiagnostic; use crate::working_directory::WorkingDirectoryDiagnostic;
use async_trait::async_trait; use async_trait::async_trait;
@@ -3313,6 +3281,14 @@ mod tests {
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
fn test_command() -> protocol::WorkerCommandEnvelope {
protocol::WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 1,
expected_worker_state_revision: 0,
}
}
#[test] #[test]
fn repository_resource_failures_keep_typed_credential_diagnostics() { fn repository_resource_failures_keep_typed_credential_diagnostics() {
let cases = [ let cases = [
@@ -3359,7 +3335,9 @@ mod tests {
protocol::Event::InternalWorker { protocol::Event::InternalWorker {
worker, worker,
revision: 1, revision: 1,
event: Box::new(protocol::Event::Status { status }), event: Box::new(protocol::Event::WorkerState {
snapshot: status.into(),
}),
} }
} }
@@ -3452,7 +3430,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: protocol::WorkerStatus::Idle, state: protocol::WorkerStatus::Idle.into(),
in_flight: protocol::InFlightSnapshot::default(), in_flight: protocol::InFlightSnapshot::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}; };
@@ -3967,7 +3945,6 @@ mod tests {
.insert(request.worker_ref.worker_id.clone(), request.context); .insert(request.worker_ref.worker_id.clone(), request.context);
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request working_directory: request
.working_directory .working_directory
.as_ref() .as_ref()
@@ -3997,7 +3974,6 @@ mod tests {
.insert(request.worker_ref.worker_id.clone(), request.context); .insert(request.worker_ref.worker_id.clone(), request.context);
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request working_directory: request
.working_directory .working_directory
.as_ref() .as_ref()
@@ -4020,7 +3996,6 @@ mod tests {
.unwrap_or_else(|| { .unwrap_or_else(|| {
WorkerExecutionResult::accepted_submission( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
"request-test", "request-test",
"test-submission", "test-submission",
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
@@ -4038,17 +4013,11 @@ mod tests {
} }
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
WorkerExecutionOperation::Stop,
WorkerExecutionRunState::Stopped,
)
} }
fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::Cancel)
WorkerExecutionOperation::Cancel,
WorkerExecutionRunState::Stopped,
)
} }
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
@@ -4374,7 +4343,9 @@ mod tests {
.send_protocol_method_scoped( .send_protocol_method_scoped(
&scope("workspace-a", "server-a"), &scope("workspace-a", "server-a"),
&workspace_b.worker_ref, &workspace_b.worker_ref,
Method::Shutdown, Method::Shutdown {
command: test_command(),
},
) )
.unwrap_err(); .unwrap_err();
assert!(matches!( assert!(matches!(
@@ -4722,11 +4693,10 @@ mod tests {
} }
#[test] #[test]
fn create_worker_uses_committed_input_ack_run_state() { fn create_worker_does_not_infer_state_from_started_submission_ack() {
let (runtime, backend) = runtime_and_backend(); let (runtime, backend) = runtime_and_backend();
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
"request-test", "request-test",
"test-submission", "test-submission",
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
@@ -4737,6 +4707,68 @@ mod tests {
let detail = runtime.create_worker(request).unwrap(); let detail = runtime.create_worker(request).unwrap();
assert_eq!(detail.status, WorkerStatus::Idle); assert_eq!(detail.status, WorkerStatus::Idle);
assert_eq!(detail.worker_state, None);
}
#[test]
fn runtime_applies_only_newer_worker_state_snapshots() {
let (runtime, _) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("state ordering"))
.unwrap();
let running = protocol::WorkerStateSnapshot {
execution_generation: 7,
revision: 3,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
};
assert!({
let mut state = runtime.lock().unwrap();
state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: running.clone(),
},
)
});
assert_eq!(
runtime
.worker_detail(&detail.worker_ref)
.unwrap()
.worker_state,
Some(running.clone())
);
assert!({
let mut state = runtime.lock().unwrap();
!state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
revision: 2,
state: protocol::WorkerState::Idle,
..running.clone()
},
},
)
});
assert!({
let mut state = runtime.lock().unwrap();
!state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
..running.clone()
},
},
)
});
let after = runtime.worker_detail(&detail.worker_ref).unwrap();
assert_eq!(after.status, WorkerStatus::Idle);
assert_eq!(after.worker_state, Some(running));
} }
#[test] #[test]
@@ -4745,7 +4777,6 @@ mod tests {
backend.preserve_commit_ack_submission_id(); backend.preserve_commit_ack_submission_id();
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
"request-test", "request-test",
"forged-submission", "forged-submission",
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
@@ -4770,7 +4801,6 @@ mod tests {
let (runtime, backend) = runtime_and_backend(); let (runtime, backend) = runtime_and_backend();
backend.set_dispatch_result(WorkerExecutionResult::accepted( backend.set_dispatch_result(WorkerExecutionResult::accepted(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
)); ));
let mut request = task_request("missing initial input commit ack"); let mut request = task_request("missing initial input commit ack");
request.initial_input = Some(WorkerInput::user("start the ticket")); request.initial_input = Some(WorkerInput::user("start the ticket"));
@@ -4898,7 +4928,7 @@ mod tests {
context_window: 128, context_window: 128,
context_tokens: 64, context_tokens: 64,
}, },
status: protocol::WorkerStatus::Running, state: protocol::WorkerStatus::Running.into(),
in_flight: protocol::InFlightSnapshot { in_flight: protocol::InFlightSnapshot {
blocks: Vec::new(), blocks: Vec::new(),
commands: Vec::new(), commands: Vec::new(),
@@ -4914,18 +4944,38 @@ mod tests {
protocol::Event::Snapshot { protocol::Event::Snapshot {
session, session,
greeting, greeting,
status, state,
.. ..
} => { } => {
assert_eq!(session.entries.len(), 1); assert_eq!(session.entries.len(), 1);
assert_eq!(session.entries[0].entry_id, "restored-log-entry"); assert_eq!(session.entries[0].entry_id, "restored-log-entry");
assert_eq!(greeting.worker_name, "live-worker"); assert_eq!(greeting.worker_name, "live-worker");
assert_eq!(status, protocol::WorkerStatus::Running); assert_eq!(state.catalog_status(), protocol::WorkerStatus::Running);
} }
other => panic!("expected snapshot, got {other:?}"), other => panic!("expected snapshot, got {other:?}"),
} }
} }
#[cfg(feature = "ws-server")]
#[test]
fn observation_snapshot_fails_closed_when_backend_snapshot_is_unavailable() {
let runtime = runtime_with_backend();
let detail = runtime
.create_worker(task_request("snapshot unavailable"))
.unwrap();
assert!(matches!(
runtime
.worker_observation_snapshot(&detail.worker_ref)
.unwrap_err(),
RuntimeError::WorkerExecutionUnavailable {
worker_id,
message,
} if worker_id == detail.worker_ref.worker_id
&& message == "authoritative Worker snapshot is unavailable"
));
}
struct InputOnlyBackend; struct InputOnlyBackend;
impl WorkerExecutionBackend for InputOnlyBackend { impl WorkerExecutionBackend for InputOnlyBackend {
@@ -4936,7 +4986,6 @@ mod tests {
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: request working_directory: request
.working_directory .working_directory
.as_ref() .as_ref()
@@ -4951,7 +5000,6 @@ mod tests {
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
WorkerExecutionResult::accepted_submission( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Idle,
"request-test", "request-test",
input input
.submission_request_id .submission_request_id
@@ -4993,7 +5041,12 @@ mod tests {
.unwrap(); .unwrap();
runtime runtime
.send_protocol_method(&detail.worker_ref, Method::Shutdown) .send_protocol_method(
&detail.worker_ref,
Method::Shutdown {
command: test_command(),
},
)
.unwrap(); .unwrap();
assert_eq!( assert_eq!(
@@ -5009,7 +5062,12 @@ mod tests {
.create_worker(task_request("restore explicitly")) .create_worker(task_request("restore explicitly"))
.unwrap(); .unwrap();
runtime runtime
.send_protocol_method(&detail.worker_ref, Method::Shutdown) .send_protocol_method(
&detail.worker_ref,
Method::Shutdown {
command: test_command(),
},
)
.unwrap(); .unwrap();
assert!(matches!( assert!(matches!(
@@ -5025,10 +5083,9 @@ mod tests {
assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]); assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
assert_eq!( let restored = runtime.worker_detail(&detail.worker_ref).unwrap();
runtime.worker_detail(&detail.worker_ref).unwrap().status, assert_eq!(restored.status, WorkerStatus::Idle);
WorkerStatus::Idle assert_eq!(restored.worker_state, None);
);
} }
#[test] #[test]
+313 -290
View File
@@ -10,8 +10,8 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::future::Future; use std::future::Future;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, mpsc}; use std::sync::{Arc, Mutex, RwLock, mpsc};
use std::time::Duration; use std::time::Duration;
use crate::auth::{ use crate::auth::{
@@ -25,8 +25,8 @@ use crate::catalog::{
}; };
use crate::execution::{ use crate::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest,
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, WorkerExecutionSpawnResult,
}; };
use crate::identity::WorkerRef; use crate::identity::WorkerRef;
use crate::interaction::{WorkerInput, WorkerInputKind}; use crate::interaction::{WorkerInput, WorkerInputKind};
@@ -38,7 +38,28 @@ use crate::working_directory::{
WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer,
}; };
use async_trait::async_trait; use async_trait::async_trait;
use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus}; #[cfg(test)]
use protocol::WorkerStatus;
use protocol::{Event, Method, Segment, WorkerCommandEnvelope};
static NEXT_INTERNAL_COMMAND_ID: AtomicU64 = AtomicU64::new(1);
fn next_internal_command(
state: &RwLock<protocol::WorkerStateSnapshot>,
) -> Result<WorkerCommandEnvelope, String> {
let snapshot = state
.read()
.map_err(|_| "worker state lock is poisoned".to_string())?
.clone();
let floor = snapshot.last_command_id.saturating_add(1);
let command_id = NEXT_INTERNAL_COMMAND_ID
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
Some(current.max(floor).saturating_add(1))
})
.unwrap_or(floor)
.max(floor);
Ok(WorkerCommandEnvelope::for_snapshot(command_id, &snapshot))
}
use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore}; use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
#[cfg(test)] #[cfg(test)]
use session_store::{FsStore, FsWorkerStore}; use session_store::{FsStore, FsWorkerStore};
@@ -172,7 +193,7 @@ impl WorkerObservationProvider for RuntimeGrantedWorkerObservationProvider {
}, },
display_name: grant.worker_id.clone(), display_name: grant.worker_id.clone(),
relation: "granted_peer".to_string(), relation: "granted_peer".to_string(),
status: format!("{:?}", state.get_status()).to_lowercase(), status: format!("{:?}", state.catalog_status()).to_lowercase(),
}); });
} }
subjects.sort_by(|left, right| left.subject.cmp(&right.subject)); subjects.sort_by(|left, right| left.subject.cmp(&right.subject));
@@ -1174,10 +1195,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
} }
} }
#[derive(Clone)]
struct RuntimeWorkerExecution { struct RuntimeWorkerExecution {
handle: WorkerHandle, handle: WorkerHandle,
shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>, shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>,
busy: Arc<AtomicBool>, worker_state: Arc<RwLock<protocol::WorkerStateSnapshot>>,
workspace_client: Option<Arc<dyn WorkspaceClient>>, workspace_client: Option<Arc<dyn WorkspaceClient>>,
} }
@@ -1275,7 +1297,7 @@ where
) -> Result< ) -> Result<
( (
WorkerHandle, WorkerHandle,
Arc<AtomicBool>, Arc<RwLock<protocol::WorkerStateSnapshot>>,
Option<Arc<dyn WorkspaceClient>>, Option<Arc<dyn WorkspaceClient>>,
), ),
WorkerExecutionResult, WorkerExecutionResult,
@@ -1301,7 +1323,7 @@ where
.map(|execution| { .map(|execution| {
( (
execution.handle.clone(), execution.handle.clone(),
execution.busy.clone(), execution.worker_state.clone(),
execution.workspace_client.clone(), execution.workspace_client.clone(),
) )
}) })
@@ -1318,7 +1340,6 @@ where
operation: WorkerExecutionOperation, operation: WorkerExecutionOperation,
worker: WorkerHandle, worker: WorkerHandle,
method: Method, method: Method,
accepted_run_state: WorkerExecutionRunState,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
self.run_on_adapter_runtime(async move { self.run_on_adapter_runtime(async move {
worker worker
@@ -1326,7 +1347,7 @@ where
.await .await
.map_err(|err| format!("failed to send Worker method: {err}")) .map_err(|err| format!("failed to send Worker method: {err}"))
}) })
.map(|_| WorkerExecutionResult::accepted(operation, accepted_run_state)) .map(|_| WorkerExecutionResult::accepted(operation))
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
} }
@@ -1336,7 +1357,6 @@ where
worker: WorkerHandle, worker: WorkerHandle,
method: Method, method: Method,
submission_request_id: String, submission_request_id: String,
accepted_run_state: WorkerExecutionRunState,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
let request_id = submission_request_id.clone(); let request_id = submission_request_id.clone();
self.run_on_adapter_runtime(async move { self.run_on_adapter_runtime(async move {
@@ -1395,7 +1415,6 @@ where
.map(|(submission_id, disposition)| { .map(|(submission_id, disposition)| {
WorkerExecutionResult::accepted_submission( WorkerExecutionResult::accepted_submission(
operation, operation,
accepted_run_state,
submission_request_id, submission_request_id,
submission_id, submission_id,
disposition, disposition,
@@ -1414,41 +1433,31 @@ where
working_directory: Option<WorkingDirectoryBinding>, working_directory: Option<WorkingDirectoryBinding>,
workspace_client: Option<Arc<dyn WorkspaceClient>>, workspace_client: Option<Arc<dyn WorkspaceClient>>,
) -> WorkerExecutionSpawnResult { ) -> WorkerExecutionSpawnResult {
let busy = Arc::new(AtomicBool::new(false)); let worker_state = Arc::new(RwLock::new(handle.shared_state.snapshot()));
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
{ {
let streams = subscribe_worker_protocol_session(&handle); let streams = subscribe_worker_protocol_session(&handle);
let mut events = streams.events; let mut events = streams.events;
let mut entry_events = streams.log_entries; let mut entry_events = streams.log_entries;
let bridge_busy = busy.clone(); let bridge_worker_state = worker_state.clone();
if let Err(message) = self.spawn_on_adapter_runtime(async move { if let Err(message) = self.spawn_on_adapter_runtime(async move {
loop { loop {
tokio::select! { tokio::select! {
event = events.recv() => { event = events.recv() => {
match event { match event {
Ok(event) => { Ok(mut event) => {
let next_busy = match &event { match apply_protocol_worker_state(&bridge_worker_state, &mut event) {
Event::InvokeStart { .. } Ok(true) => {
| Event::Status { let _ = bridge_context.publish_protocol_event(event);
status: WorkerStatus::Running,
} => Some(true),
Event::RunEnd { .. }
| Event::Error {
code: ErrorCode::NotPaused,
..
} }
| Event::Status { Ok(false) => {}
status: Err(message) => {
WorkerStatus::Idle let _ = bridge_context.publish_protocol_event(Event::Error {
| WorkerStatus::Paused code: protocol::ErrorCode::Internal,
| WorkerStatus::Stopped, message: format!("worker state stream rejected: {message}"),
});
break;
} }
| Event::Shutdown => Some(false),
_ => None,
};
let _ = bridge_context.publish_protocol_event(event);
if let Some(next_busy) = next_busy {
bridge_busy.store(next_busy, Ordering::SeqCst);
} }
} }
Err(broadcast::error::RecvError::Lagged(_)) => continue, Err(broadcast::error::RecvError::Lagged(_)) => continue,
@@ -1493,14 +1502,13 @@ where
RuntimeWorkerExecution { RuntimeWorkerExecution {
handle, handle,
shutdown, shutdown,
busy, worker_state,
workspace_client, workspace_client,
}, },
); );
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()), handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()),
run_state: WorkerExecutionRunState::Idle,
working_directory: working_directory.map(|binding| binding.status()), working_directory: working_directory.map(|binding| binding.status()),
} }
} }
@@ -1516,45 +1524,28 @@ impl<F> Drop for WorkerRuntimeExecutionBackend<F> {
} }
} }
fn method_starts_turn(method: &Method) -> bool { fn apply_protocol_worker_state(
matches!( current: &Arc<RwLock<protocol::WorkerStateSnapshot>>,
method, event: &mut Event,
Method::Submit { .. } ) -> Result<bool, String> {
| Method::SubmitTracked { .. } let (incoming, replace_stale) = match event {
| Method::Notify { auto_run: true, .. } Event::WorkerState { snapshot } => (snapshot, false),
| Method::NotifyTracked { auto_run: true, .. } Event::Snapshot { state, .. } => (state, true),
| Method::Resume Event::CommandAcknowledged { acknowledgement } => (&mut acknowledgement.state, true),
| Method::Compact _ => return Ok(true),
) };
} let mut current = current
.write()
fn method_can_start_turn_from_status(method: &Method, status: WorkerStatus) -> bool { .map_err(|_| "worker state projection lock is poisoned".to_string())?;
match method { match protocol::apply_worker_state_snapshot(&mut current, incoming) {
Method::Resume => matches!(status, WorkerStatus::Idle | WorkerStatus::Paused), Ok(protocol::WorkerStateSnapshotApply::Applied)
_ => status == WorkerStatus::Idle, | Ok(protocol::WorkerStateSnapshotApply::Duplicate) => Ok(true),
} Ok(protocol::WorkerStateSnapshotApply::Stale) if replace_stale => {
} *incoming = current.clone();
Ok(true)
fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExecutionRunState {
match status {
WorkerStatus::Running => WorkerExecutionRunState::Busy,
WorkerStatus::Idle if auto_run => WorkerExecutionRunState::Busy,
WorkerStatus::Idle | WorkerStatus::Paused | WorkerStatus::Stopped => {
WorkerExecutionRunState::Idle
} }
} Ok(protocol::WorkerStateSnapshotApply::Stale) => Ok(false),
} Err(error) => Err(error.to_string()),
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
match method {
Method::Submit { .. }
| Method::SubmitTracked { .. }
| Method::Notify { auto_run: true, .. }
| Method::NotifyTracked { auto_run: true, .. }
| Method::Resume
| Method::Compact => WorkerExecutionRunState::Busy,
Method::Shutdown => WorkerExecutionRunState::Stopped,
_ => WorkerExecutionRunState::Idle,
} }
} }
@@ -1883,7 +1874,7 @@ where
handle: &WorkerExecutionHandle, handle: &WorkerExecutionHandle,
input: WorkerInput, input: WorkerInput,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
let (worker, busy, _workspace_client) = match self.get_execution(handle) { let (worker, worker_state, _workspace_client) = match self.get_execution(handle) {
Ok(execution) => execution, Ok(execution) => execution,
Err(mut result) => { Err(mut result) => {
result.operation = WorkerExecutionOperation::Input; result.operation = WorkerExecutionOperation::Input;
@@ -1892,16 +1883,10 @@ where
}; };
if input.kind == WorkerInputKind::Notify { if input.kind == WorkerInputKind::Notify {
let status = worker.shared_state.get_status();
let accepted_run_state = accepted_notify_run_state(status, true);
let claimed_here = status == WorkerStatus::Idle
&& busy
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_ok();
let notification_request_id = input let notification_request_id = input
.submission_request_id .submission_request_id
.unwrap_or_else(protocol::new_submission_request_id); .unwrap_or_else(protocol::new_submission_request_id);
let result = self.send_method( return self.send_method(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
worker, worker,
Method::NotifyTracked { Method::NotifyTracked {
@@ -1912,25 +1897,20 @@ where
operation_id: notification_request_id, operation_id: notification_request_id,
}, },
}, },
accepted_run_state,
); );
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
{
busy.store(false, Ordering::SeqCst);
}
return result;
} }
let is_user_submit = input.kind == WorkerInputKind::User; if input.kind == WorkerInputKind::Compact {
let status = worker.shared_state.get_status(); let command = match next_internal_command(&worker_state) {
let claimed_here = status == WorkerStatus::Idle Ok(command) => command,
&& busy Err(error) => {
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) return WorkerExecutionResult::errored(WorkerExecutionOperation::Input, error);
.is_ok(); }
if !is_user_submit && !claimed_here { };
return WorkerExecutionResult::busy( return self.send_method(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
"Worker is already running", worker,
Method::Compact { command },
); );
} }
@@ -1940,7 +1920,6 @@ where
.submission_request_id .submission_request_id
.filter(|submission_id| !submission_id.trim().is_empty()) .filter(|submission_id| !submission_id.trim().is_empty())
else { else {
busy.store(false, Ordering::SeqCst);
return WorkerExecutionResult::rejected( return WorkerExecutionResult::rejected(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
"Runtime user input is missing its internal submission id", "Runtime user input is missing its internal submission id",
@@ -1960,9 +1939,9 @@ where
) )
} }
WorkerInputKind::Notify => { WorkerInputKind::Notify => {
unreachable!("Notify input is dispatched before the turn-start busy guard") unreachable!("Notify input is dispatched before ordinary input mapping")
} }
WorkerInputKind::Compact => (Method::Compact, None), WorkerInputKind::Compact => unreachable!("compact input is dispatched above"),
WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None), WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None),
WorkerInputKind::RegisterPeer => ( WorkerInputKind::RegisterPeer => (
Method::RegisterPeer { Method::RegisterPeer {
@@ -1971,40 +1950,18 @@ where
None, None,
), ),
}; };
let accepted_run_state = match method {
Method::Submit { .. }
| Method::SubmitTracked { .. }
| Method::Notify { .. }
| Method::NotifyTracked { .. }
| Method::Compact => WorkerExecutionRunState::Busy,
_ => WorkerExecutionRunState::Idle,
};
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
let waits_for_submission_acceptance = submission_request_id.is_some(); let waits_for_submission_acceptance = submission_request_id.is_some();
let result = if waits_for_submission_acceptance { if waits_for_submission_acceptance {
self.send_submit_and_wait_for_acceptance( self.send_submit_and_wait_for_acceptance(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
worker, worker,
method, method,
submission_request_id.expect("Submit must have a submission request id"), submission_request_id.expect("Submit must have a submission request id"),
accepted_run_state,
) )
} else { } else {
self.send_method( self.send_method(WorkerExecutionOperation::Input, worker, method)
WorkerExecutionOperation::Input,
worker,
method,
accepted_run_state,
)
};
if accepted_is_idle
|| (claimed_here
&& result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
{
busy.store(false, Ordering::SeqCst);
} }
result
} }
fn upload_file( fn upload_file(
@@ -2046,10 +2003,7 @@ where
} }
}; };
match worker.delete_uploaded_file(artifact_id) { match worker.delete_uploaded_file(artifact_id) {
Ok(_) => WorkerExecutionResult::accepted( Ok(_) => WorkerExecutionResult::accepted(WorkerExecutionOperation::DeleteUploadedFile),
WorkerExecutionOperation::DeleteUploadedFile,
WorkerExecutionRunState::Idle,
),
Err(error) => WorkerExecutionResult::rejected( Err(error) => WorkerExecutionResult::rejected(
WorkerExecutionOperation::DeleteUploadedFile, WorkerExecutionOperation::DeleteUploadedFile,
format!("uploaded_file_delete_rejected: {error}"), format!("uploaded_file_delete_rejected: {error}"),
@@ -2062,7 +2016,7 @@ where
handle: &WorkerExecutionHandle, handle: &WorkerExecutionHandle,
method: Method, method: Method,
) -> WorkerExecutionResult { ) -> WorkerExecutionResult {
let (worker, busy, _workspace_client) = match self.get_execution(handle) { let (worker, _worker_state, _workspace_client) = match self.get_execution(handle) {
Ok(execution) => execution, Ok(execution) => execution,
Err(mut result) => { Err(mut result) => {
result.operation = WorkerExecutionOperation::ProtocolMethod; result.operation = WorkerExecutionOperation::ProtocolMethod;
@@ -2070,59 +2024,7 @@ where
} }
}; };
if let Some(auto_run) = match &method { self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method)
Method::Notify { auto_run, .. } | Method::NotifyTracked { auto_run, .. } => {
Some(*auto_run)
}
_ => None,
} {
let status = worker.shared_state.get_status();
let accepted_run_state = accepted_notify_run_state(status, auto_run);
let claimed_here = status == WorkerStatus::Idle
&& auto_run
&& busy
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_ok();
let result = self.send_method(
WorkerExecutionOperation::ProtocolMethod,
worker,
method,
accepted_run_state,
);
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
{
busy.store(false, Ordering::SeqCst);
}
return result;
}
let starts_turn = method_starts_turn(&method);
if starts_turn
&& (!method_can_start_turn_from_status(&method, worker.shared_state.get_status())
|| busy
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err())
{
return WorkerExecutionResult::busy(
WorkerExecutionOperation::ProtocolMethod,
"Worker is already running; runtime adapter v0 does not queue protocol methods",
);
}
let accepted_run_state = accepted_run_state_for_method(&method);
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
let result = self.send_method(
WorkerExecutionOperation::ProtocolMethod,
worker,
method,
accepted_run_state,
);
if (starts_turn && accepted_is_idle)
|| (starts_turn && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
{
busy.store(false, Ordering::SeqCst);
}
result
} }
fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
@@ -2137,7 +2039,7 @@ where
); );
} }
let execution = match self.workers.lock() { let execution = match self.workers.lock() {
Ok(mut workers) => workers.remove(handle.worker_ref()), Ok(workers) => workers.get(handle.worker_ref()).cloned(),
Err(_) => { Err(_) => {
return WorkerExecutionResult::errored( return WorkerExecutionResult::errored(
WorkerExecutionOperation::Stop, WorkerExecutionOperation::Stop,
@@ -2153,48 +2055,73 @@ where
}; };
let artifact_cleanup = execution.handle.clone(); let artifact_cleanup = execution.handle.clone();
let shutdown = execution.shutdown.clone(); let shutdown = execution.shutdown.clone();
let command = match next_internal_command(&execution.worker_state) {
Ok(command) => command,
Err(error) => {
return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, error);
}
};
let result = self.send_method( let result = self.send_method(
WorkerExecutionOperation::Stop, WorkerExecutionOperation::Stop,
execution.handle, execution.handle.clone(),
Method::Shutdown, Method::Shutdown { command },
WorkerExecutionRunState::Stopped,
); );
if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted {
return result; return result;
} }
match self.run_on_adapter_runtime(async move { let shutdown_wait = self.run_on_adapter_runtime(async move {
let receiver = shutdown.lock().await.take(); let mut guard = shutdown.lock().await;
if let Some(receiver) = receiver { let Some(mut receiver) = guard.take() else {
receiver return Ok(());
.await };
.map_err(|_| "Worker shutdown completion channel closed".to_string())?; match tokio::time::timeout(Duration::from_secs(5), &mut receiver).await {
Ok(Ok(())) => Ok(()),
Ok(Err(_)) => Err("Worker shutdown completion channel closed".to_string()),
Err(_) => {
*guard = Some(receiver);
Err("Worker shutdown confirmation timed out; stop remains retryable".into())
}
} }
Ok(()) });
}) { if let Err(message) = shutdown_wait {
Ok(()) => match artifact_cleanup.delete_uncommitted_uploaded_files() { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message);
Ok(_) => result, }
Err(error) => WorkerExecutionResult::errored( if let Err(error) = artifact_cleanup.delete_uncommitted_uploaded_files() {
WorkerExecutionOperation::Stop, return WorkerExecutionResult::errored(
format!("uploaded_file_cleanup_failed: {error}"), WorkerExecutionOperation::Stop,
), format!("uploaded_file_cleanup_failed: {error}"),
}, );
Err(message) => WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message), }
match self.workers.lock() {
Ok(mut workers) => {
workers.remove(handle.worker_ref());
result
}
Err(_) => WorkerExecutionResult::errored(
WorkerExecutionOperation::Stop,
"worker adapter registry lock is poisoned after shutdown",
),
} }
} }
fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
let (worker, _busy, _workspace_client) = match self.get_execution(handle) { let (worker, worker_state, _workspace_client) = match self.get_execution(handle) {
Ok(execution) => execution, Ok(execution) => execution,
Err(mut result) => { Err(mut result) => {
result.operation = WorkerExecutionOperation::Cancel; result.operation = WorkerExecutionOperation::Cancel;
return result; return result;
} }
}; };
let command = match next_internal_command(&worker_state) {
Ok(command) => command,
Err(error) => {
return WorkerExecutionResult::errored(WorkerExecutionOperation::Cancel, error);
}
};
self.send_method( self.send_method(
WorkerExecutionOperation::Cancel, WorkerExecutionOperation::Cancel,
worker, worker,
Method::Cancel, Method::Cancel { command },
WorkerExecutionRunState::Idle,
) )
} }
@@ -2259,6 +2186,79 @@ mod tests {
use manifest::{Scope, WorkerManifest}; use manifest::{Scope, WorkerManifest};
use session_store::{LogEntry, WorkerMetadataStore}; use session_store::{LogEntry, WorkerMetadataStore};
fn test_command() -> WorkerCommandEnvelope {
WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 1,
expected_worker_state_revision: 0,
}
}
fn adapter_command(
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
worker_ref: &WorkerRef,
) -> WorkerCommandEnvelope {
let workers = backend.workers.lock().unwrap();
let state = workers
.get(worker_ref)
.expect("worker execution")
.worker_state
.read()
.unwrap()
.clone();
WorkerCommandEnvelope::for_snapshot(state.last_command_id.saturating_add(1), &state)
}
#[test]
fn protocol_bridge_applies_state_and_acknowledgement_monotonically() {
let running = protocol::WorkerStateSnapshot {
execution_generation: 4,
revision: 3,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
};
let current = Arc::new(RwLock::new(running.clone()));
let mut stale = Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
revision: 2,
state: protocol::WorkerState::Idle,
..running.clone()
},
};
assert!(!apply_protocol_worker_state(&current, &mut stale).unwrap());
assert_eq!(*current.read().unwrap(), running);
let paused = protocol::WorkerStateSnapshot {
revision: 4,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Paused,
)),
last_command_id: 3,
..running.clone()
};
let mut acknowledgement = Event::CommandAcknowledged {
acknowledgement: protocol::WorkerCommandAcknowledgement {
command_id: 3,
command: protocol::WorkerCommandKind::Pause,
disposition: protocol::WorkerCommandDisposition::Accepted,
state: paused.clone(),
},
};
assert!(apply_protocol_worker_state(&current, &mut acknowledgement).unwrap());
assert_eq!(*current.read().unwrap(), paused);
let mut conflict = Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
..paused.clone()
},
};
assert!(apply_protocol_worker_state(&current, &mut conflict).is_err());
assert_eq!(*current.read().unwrap(), paused);
}
#[test] #[test]
fn workspace_prompt_projection_notification_advances_shared_cache() { fn workspace_prompt_projection_notification_advances_shared_cache() {
let cache = WorkspacePromptProjectionCache::default(); let cache = WorkspacePromptProjectionCache::default();
@@ -2405,46 +2405,6 @@ mod tests {
assert_eq!(after_restore_workspace_id.as_deref(), Some("workspace-a")); assert_eq!(after_restore_workspace_id.as_deref(), Some("workspace-a"));
} }
#[test]
fn notify_run_state_allows_running_worker_inbox_delivery() {
assert_eq!(
accepted_notify_run_state(WorkerStatus::Running, true),
WorkerExecutionRunState::Busy
);
assert_eq!(
accepted_notify_run_state(WorkerStatus::Idle, true),
WorkerExecutionRunState::Busy
);
assert_eq!(
accepted_notify_run_state(WorkerStatus::Idle, false),
WorkerExecutionRunState::Idle
);
assert_eq!(
accepted_notify_run_state(WorkerStatus::Paused, true),
WorkerExecutionRunState::Idle
);
}
#[test]
fn resume_turn_claim_accepts_paused_and_idle_but_not_running_status() {
assert!(method_can_start_turn_from_status(
&Method::Resume,
WorkerStatus::Paused
));
assert!(method_can_start_turn_from_status(
&Method::Resume,
WorkerStatus::Idle
));
assert!(!method_can_start_turn_from_status(
&Method::Resume,
WorkerStatus::Running
));
assert!(!method_can_start_turn_from_status(
&Method::Compact,
WorkerStatus::Paused
));
}
#[derive(Clone)] #[derive(Clone)]
enum MockResponse { enum MockResponse {
Complete(Vec<LlmEvent>), Complete(Vec<LlmEvent>),
@@ -2645,28 +2605,53 @@ mod tests {
.collect() .collect()
} }
fn wait_for_adapter_command(
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
worker_ref: &WorkerRef,
expected_command_id: u64,
) {
let deadline = std::time::Instant::now() + Duration::from_secs(5);
loop {
let observed = {
let workers = backend.workers.lock().unwrap();
workers
.get(worker_ref)
.expect("live Worker execution")
.worker_state
.read()
.unwrap()
.last_command_id
};
if observed >= expected_command_id {
return;
}
assert!(
std::time::Instant::now() < deadline,
"timed out waiting for adapter command {expected_command_id}; last observed={observed}",
);
std::thread::sleep(Duration::from_millis(10));
}
}
fn wait_for_adapter_state( fn wait_for_adapter_state(
backend: &WorkerRuntimeExecutionBackend<MockFactory>, backend: &WorkerRuntimeExecutionBackend<MockFactory>,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
expected_status: WorkerStatus, expected_status: WorkerStatus,
expected_busy: bool,
) { ) {
let deadline = std::time::Instant::now() + Duration::from_secs(5); let deadline = std::time::Instant::now() + Duration::from_secs(5);
loop { loop {
let observed = { let observed = {
let workers = backend.workers.lock().unwrap(); let workers = backend.workers.lock().unwrap();
let execution = workers.get(worker_ref).expect("live Worker execution"); let execution = workers.get(worker_ref).expect("live Worker execution");
( let projected = execution.worker_state.read().unwrap().catalog_status();
execution.handle.shared_state.get_status(), (execution.handle.shared_state.catalog_status(), projected)
execution.busy.load(Ordering::SeqCst),
)
}; };
if observed == (expected_status, expected_busy) { if observed == (expected_status, expected_status) {
return; return;
} }
assert!( assert!(
std::time::Instant::now() < deadline, std::time::Instant::now() < deadline,
"timed out waiting for adapter state {expected_status:?}, busy={expected_busy}; last observed status={:?}, busy={}", "timed out waiting for adapter state {expected_status:?}; last observed controller={:?}, projected={:?}",
observed.0, observed.0,
observed.1, observed.1,
); );
@@ -3169,13 +3154,19 @@ mod tests {
.expect("in-process restore must not bind the overlong Unix socket path"); .expect("in-process restore must not bind the overlong Unix socket path");
assert_eq!( assert_eq!(
controller.handle.shared_state.get_status(), controller.handle.shared_state.catalog_status(),
WorkerStatus::Idle WorkerStatus::Idle
); );
assert!(!socket_path.exists()); assert!(!socket_path.exists());
assert!(run_dir.join("worker.out.log").is_file()); assert!(run_dir.join("worker.out.log").is_file());
assert!(run_dir.join("worker.err.log").is_file()); assert!(run_dir.join("worker.err.log").is_file());
controller.handle.send(Method::Shutdown).await.unwrap(); controller
.handle
.send(Method::Shutdown {
command: test_command(),
})
.await
.unwrap();
if let Some(receiver) = controller.shutdown.lock().await.take() { if let Some(receiver) = controller.shutdown.lock().await.take() {
receiver.await.unwrap(); receiver.await.unwrap();
} }
@@ -3289,7 +3280,9 @@ mod tests {
backend backend
.run_on_adapter_runtime(async move { .run_on_adapter_runtime(async move {
handle handle
.send(Method::Shutdown) .send(Method::Shutdown {
command: test_command(),
})
.await .await
.map_err(|error| error.to_string())?; .map_err(|error| error.to_string())?;
if let Some(receiver) = shutdown.lock().await.take() { if let Some(receiver) = shutdown.lock().await.take() {
@@ -3614,6 +3607,7 @@ mod tests {
#[test] #[test]
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
#[serial_test::serial(worker_allocation)]
fn adapter_resumes_paused_turn_once_and_preserves_idle_not_paused_error() { fn adapter_resumes_paused_turn_once_and_preserves_idle_not_paused_error() {
let hanging_events = || simple_text_events().into_iter().take(2).collect::<Vec<_>>(); let hanging_events = || simple_text_events().into_iter().take(2).collect::<Vec<_>>();
let client = MockClient::sequential(vec![ let client = MockClient::sequential(vec![
@@ -3646,62 +3640,91 @@ mod tests {
runtime runtime
.send_input(&detail.worker_ref, WorkerInput::user("pause and resume")) .send_input(&detail.worker_ref, WorkerInput::user("pause and resume"))
.expect("start initial turn"); .expect("start initial turn");
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running);
let running_resume = runtime let running_resume = adapter_command(&backend, &detail.worker_ref);
.send_protocol_method(&detail.worker_ref, Method::Resume) runtime
.expect_err("Resume while Running must be rejected"); .send_protocol_method(
assert!( &detail.worker_ref,
running_resume Method::Resume {
.to_string() command: running_resume,
.contains("does not queue protocol methods"), },
"unexpected Running Resume error: {running_resume}" )
); .expect("running Resume is forwarded for controller admission");
wait_for_adapter_command(&backend, &detail.worker_ref, running_resume.command_id);
runtime runtime
.send_protocol_method(&detail.worker_ref, Method::Pause) .send_protocol_method(
&detail.worker_ref,
Method::Pause {
command: adapter_command(&backend, &detail.worker_ref),
},
)
.expect("pause initial turn"); .expect("pause initial turn");
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused);
runtime runtime
.send_protocol_method(&detail.worker_ref, Method::Resume) .send_protocol_method(
&detail.worker_ref,
Method::Resume {
command: adapter_command(&backend, &detail.worker_ref),
},
)
.expect("resume paused turn"); .expect("resume paused turn");
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running);
let duplicate_resume = runtime let duplicate_resume = adapter_command(&backend, &detail.worker_ref);
.send_protocol_method(&detail.worker_ref, Method::Resume) runtime
.expect_err("duplicate Resume must be rejected"); .send_protocol_method(
assert!( &detail.worker_ref,
duplicate_resume Method::Resume {
.to_string() command: duplicate_resume,
.contains("does not queue protocol methods"), },
"unexpected duplicate Resume error: {duplicate_resume}" )
); .expect("duplicate Resume is forwarded for controller admission");
wait_for_adapter_command(&backend, &detail.worker_ref, duplicate_resume.command_id);
runtime runtime
.send_protocol_method(&detail.worker_ref, Method::Pause) .send_protocol_method(
&detail.worker_ref,
Method::Pause {
command: adapter_command(&backend, &detail.worker_ref),
},
)
.expect("pause resumed turn"); .expect("pause resumed turn");
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused);
runtime runtime
.send_protocol_method(&detail.worker_ref, Method::Resume) .send_protocol_method(
&detail.worker_ref,
Method::Resume {
command: adapter_command(&backend, &detail.worker_ref),
},
)
.expect("resume paused turn a second time"); .expect("resume paused turn a second time");
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle);
assert_eq!(call_count.load(Ordering::SeqCst), 3); assert_eq!(call_count.load(Ordering::SeqCst), 3);
let idle_resume = adapter_command(&backend, &detail.worker_ref);
runtime runtime
.send_protocol_method(&detail.worker_ref, Method::Resume) .send_protocol_method(
&detail.worker_ref,
Method::Resume {
command: idle_resume,
},
)
.expect("Idle Resume preserves controller NotPaused semantics"); .expect("Idle Resume preserves controller NotPaused semantics");
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); wait_for_adapter_command(&backend, &detail.worker_ref, idle_resume.command_id);
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle);
let events = runtime let events = runtime
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
.expect("read protocol events"); .expect("read protocol events");
assert!(events.iter().any(|event| { assert!(events.iter().any(|event| {
matches!( matches!(
&event.payload, &event.payload,
Event::Error { Event::CommandAcknowledged { acknowledgement }
code: protocol::ErrorCode::NotPaused, if acknowledgement.command == protocol::WorkerCommandKind::Resume
.. && acknowledgement.disposition
} == protocol::WorkerCommandDisposition::InvalidState
) )
})); }));
assert_eq!(call_count.load(Ordering::SeqCst), 3); assert_eq!(call_count.load(Ordering::SeqCst), 3);
+644 -91
View File
@@ -1,3 +1,4 @@
use std::collections::VecDeque;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
@@ -16,7 +17,7 @@ use crate::ipc::notify_buffer::NotifyBuffer;
use crate::ipc::server::SocketServer; use crate::ipc::server::SocketServer;
use crate::runtime::dir::RuntimeDir; use crate::runtime::dir::RuntimeDir;
use crate::segment_log_sink::SegmentLogSink; use crate::segment_log_sink::SegmentLogSink;
use crate::shared_state::WorkerSharedState; use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState};
use crate::shutdown_after_idle::{ use crate::shutdown_after_idle::{
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role, ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
take_shutdown_request_after_status, take_shutdown_request_after_status,
@@ -28,7 +29,9 @@ use protocol::{
AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent, AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent,
CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus, CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus,
CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice, CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice,
ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, WorkerStatus, ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef,
WorkerBusyState, WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope,
WorkerCommandKind, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStatus,
}; };
use workdir::{ use workdir::{
CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot, CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot,
@@ -138,7 +141,7 @@ impl WorkerHandle {
let event = Event::Snapshot { let event = Event::Snapshot {
session, session,
greeting: self.shared_state.greeting.clone(), greeting: self.shared_state.greeting.clone(),
status: self.shared_state.get_status(), state: self.shared_state.snapshot(),
in_flight, in_flight,
internal_workers: self.spawned_registry.internal_worker_snapshots(), internal_workers: self.spawned_registry.internal_worker_snapshots(),
}; };
@@ -178,15 +181,101 @@ impl WorkerHandle {
} }
} }
fn command_admission_disposition(
admission: WorkerCommandAdmission,
) -> Result<(), WorkerCommandDisposition> {
match admission {
WorkerCommandAdmission::Accepted => Ok(()),
WorkerCommandAdmission::Retry | WorkerCommandAdmission::StaleCommandId => {
Err(WorkerCommandDisposition::StaleCommandId)
}
WorkerCommandAdmission::Conflict => Err(WorkerCommandDisposition::Conflict),
WorkerCommandAdmission::ExecutionGenerationMismatch => {
Err(WorkerCommandDisposition::StaleExecutionGeneration)
}
WorkerCommandAdmission::StateRevisionMismatch => {
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
}
}
}
fn validate_command(
envelope: WorkerCommandEnvelope,
kind: WorkerCommandKind,
shared_state: &WorkerSharedState,
) -> Result<(), WorkerCommandDisposition> {
command_admission_disposition(shared_state.admit_command(envelope, kind, true))
}
fn validate_shutdown_command(
envelope: WorkerCommandEnvelope,
shared_state: &WorkerSharedState,
) -> Result<(), WorkerCommandDisposition> {
match shared_state.admit_command(envelope, WorkerCommandKind::Shutdown, false) {
WorkerCommandAdmission::Accepted | WorkerCommandAdmission::Retry => Ok(()),
admission => command_admission_disposition(admission),
}
}
fn acknowledge_command(
working_event_tx: &broadcast::Sender<Event>,
shared_state: &WorkerSharedState,
command_id: u64,
command: WorkerCommandKind,
disposition: WorkerCommandDisposition,
) {
shared_state.complete_command(command_id, command, disposition);
let _ = working_event_tx.send(Event::CommandAcknowledged {
acknowledgement: WorkerCommandAcknowledgement {
command_id,
command,
disposition,
state: shared_state.snapshot(),
},
});
}
fn reject_invalid_command_state(
working_event_tx: &broadcast::Sender<Event>,
shared_state: &WorkerSharedState,
envelope: WorkerCommandEnvelope,
command: WorkerCommandKind,
) {
acknowledge_command(
working_event_tx,
shared_state,
envelope.command_id,
command,
WorkerCommandDisposition::InvalidState,
);
}
async fn set_controller_state(
shared_state: &Arc<WorkerSharedState>,
runtime_dir: &RuntimeDir,
working_event_tx: &broadcast::Sender<Event>,
state: WorkerState,
) -> protocol::WorkerStateSnapshot {
let snapshot = shared_state.transition(state);
let _ = runtime_dir.write_status(shared_state).await;
let _ = working_event_tx.send(Event::WorkerState {
snapshot: snapshot.clone(),
});
snapshot
}
async fn set_controller_status( async fn set_controller_status(
shared_state: &Arc<WorkerSharedState>, shared_state: &Arc<WorkerSharedState>,
runtime_dir: &RuntimeDir, runtime_dir: &RuntimeDir,
working_event_tx: &broadcast::Sender<Event>, working_event_tx: &broadcast::Sender<Event>,
status: WorkerStatus, status: WorkerStatus,
) { ) {
shared_state.set_status(status); let state = match status {
let _ = runtime_dir.write_status(shared_state).await; WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
let _ = working_event_tx.send(Event::Status { status }); WorkerStatus::Running => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
};
set_controller_state(shared_state, runtime_dir, working_event_tx, state).await;
} }
async fn finish_controller_run<C, St>( async fn finish_controller_run<C, St>(
@@ -660,12 +749,24 @@ impl WorkerController {
// === 4. Initial runtime files + WorkerSharedState + WorkerHandle + // === 4. Initial runtime files + WorkerSharedState + WorkerHandle +
// SocketServer === // SocketServer ===
let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default(); let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default();
worker
.recover_unfinished_compaction()
.await
.map_err(|error| std::io::Error::other(error.to_string()))?;
let greeting = build_greeting(&worker); let greeting = build_greeting(&worker);
let shared_state = Arc::new(WorkerSharedState::new( let execution_generation = runtime_dir
.path()
.file_name()
.and_then(|name| name.to_str())
.and_then(|name| name.parse::<u64>().ok())
.filter(|generation| *generation > 0)
.unwrap_or(1);
let shared_state = Arc::new(WorkerSharedState::new_with_generation(
worker.manifest().worker.name.clone(), worker.manifest().worker.name.clone(),
worker.segment_id(), worker.segment_id(),
manifest_toml.clone(), manifest_toml.clone(),
greeting, greeting,
execution_generation,
)); ));
if let Some(fs_for_view) = fs_for_view { if let Some(fs_for_view) = fs_for_view {
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view)); shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view));
@@ -1447,8 +1548,9 @@ async fn controller_loop<C, St>(
} }
}; };
loop { let mut deferred_methods = VecDeque::new();
// Top-of-iteration: if an event handler staged a run, fire it
'controller: loop {
// here so the status flip → drive_turn → finish sequence lives // here so the status flip → drive_turn → finish sequence lives
// in one place, regardless of which Method caused it. // in one place, regardless of which Method caused it.
if let Some(run) = pending.take() { if let Some(run) = pending.take() {
@@ -1599,9 +1701,13 @@ async fn controller_loop<C, St>(
continue; continue;
} }
let method = match method_rx.recv().await { let method = if let Some(method) = deferred_methods.pop_front() {
Some(m) => m, method
None => break, } else {
match method_rx.recv().await {
Some(method) => method,
None => break,
}
}; };
match method { match method {
@@ -1799,7 +1905,7 @@ async fn controller_loop<C, St>(
expected_revision, expected_revision,
expected_head_id, expected_head_id,
} => { } => {
if shared_state.get_status() != WorkerStatus::Idle { if shared_state.catalog_status() != WorkerStatus::Idle {
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest, code: ErrorCode::InvalidRequest,
message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(), message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(),
@@ -1826,88 +1932,266 @@ async fn controller_loop<C, St>(
} }
} }
} }
Method::Resume => { Method::Resume { command } => {
if shared_state.get_status() != WorkerStatus::Paused { if let Err(disposition) =
let _ = working_event_tx.send(Event::Error { validate_command(command, WorkerCommandKind::Resume, &shared_state)
code: ErrorCode::NotPaused, {
message: "Worker is not paused".into(), acknowledge_command(
}); &working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Resume,
disposition,
);
continue; continue;
} }
if !matches!(
shared_state.snapshot().state,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused))
) {
reject_invalid_command_state(
&working_event_tx,
&shared_state,
command,
WorkerCommandKind::Resume,
);
continue;
}
set_controller_state(
&shared_state,
&runtime_dir,
&working_event_tx,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
)
.await;
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Resume,
WorkerCommandDisposition::Accepted,
);
pending = Some(PendingRun::Resume); pending = Some(PendingRun::Resume);
} }
Method::Cancel => match shared_state.get_status() { Method::Cancel { command } => {
WorkerStatus::Paused => match worker.cancel_paused_turn() { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Cancel, &shared_state)
{
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Cancel,
disposition,
);
continue;
}
if !matches!(
shared_state.snapshot().state,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused))
) {
reject_invalid_command_state(
&working_event_tx,
&shared_state,
command,
WorkerCommandKind::Cancel,
);
continue;
}
set_controller_state(
&shared_state,
&runtime_dir,
&working_event_tx,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
)
.await;
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Cancel,
WorkerCommandDisposition::Accepted,
);
match worker.cancel_paused_turn() {
Ok(()) => { Ok(()) => {
worker.clear_in_flight_events(); worker.clear_in_flight_events();
set_controller_status( set_controller_state(
&shared_state, &shared_state,
&runtime_dir, &runtime_dir,
&working_event_tx, &working_event_tx,
WorkerStatus::Idle, WorkerState::Idle,
) )
.await; .await;
} }
Err(error) => { Err(error) => {
set_controller_state(
&shared_state,
&runtime_dir,
&working_event_tx,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
)
.await;
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
code: worker_error_code(&error), code: worker_error_code(&error),
message: error.to_string(), message: error.to_string(),
}); });
} }
},
WorkerStatus::Idle | WorkerStatus::Stopped => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::NotRunning,
message: "Worker is not running".into(),
});
}
WorkerStatus::Running => {
// Running turns receive Cancel through drive_turn; this is
// only reachable across a defensive race window.
let _ = cancel_tx.try_send(());
}
},
Method::Pause => {
// Already paused → idempotent no-op. Otherwise the
// Worker is Idle (Running turns go through `drive_turn`,
// not this outer match), so there is nothing to pause.
if shared_state.get_status() != WorkerStatus::Paused {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::NotRunning,
message: "Worker is not running".into(),
});
} }
} }
Method::Compact => match shared_state.get_status() { Method::Pause { command } => {
WorkerStatus::Idle => { if let Err(disposition) =
if let Err(error) = worker.manual_compact().await { validate_command(command, WorkerCommandKind::Pause, &shared_state)
let _ = working_event_tx.send(Event::Error { {
code: worker_error_code(&error), acknowledge_command(
message: error.to_string(), &working_event_tx,
}); &shared_state,
} command.command_id,
WorkerCommandKind::Pause,
disposition,
);
} else {
reject_invalid_command_state(
&working_event_tx,
&shared_state,
command,
WorkerCommandKind::Pause,
);
} }
WorkerStatus::Paused => { }
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::InvalidRequest,
message: "Cannot compact while the Worker is paused; resume or start a fresh turn first"
.into(),
});
}
WorkerStatus::Running | WorkerStatus::Stopped => {
let _ = working_event_tx.send(Event::Error {
code: ErrorCode::AlreadyRunning,
message:
"Worker is already executing a turn; compact can only run while idle"
.into(),
});
}
},
Method::ListRewindTargets => match shared_state.get_status() { Method::Compact { command } => {
if let Err(disposition) =
validate_command(command, WorkerCommandKind::Compact, &shared_state)
{
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Compact,
disposition,
);
continue;
}
if !matches!(shared_state.snapshot().state, WorkerState::Idle) {
reject_invalid_command_state(
&working_event_tx,
&shared_state,
command,
WorkerCommandKind::Compact,
);
continue;
}
set_controller_state(
&shared_state,
&runtime_dir,
&working_event_tx,
WorkerState::Busy(WorkerBusyState::Maintenance(
WorkerMaintenanceState::Compacting,
)),
)
.await;
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Compact,
WorkerCommandDisposition::Accepted,
);
let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
let mut shutdown_after_compaction = false;
let result = {
let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx));
loop {
tokio::select! {
result = &mut compact => break result,
method = method_rx.recv() => {
match method {
Some(Method::Cancel { command }) => {
if let Err(disposition) = validate_command(
command,
WorkerCommandKind::Cancel,
&shared_state,
) {
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Cancel,
disposition,
);
continue;
}
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Cancel,
WorkerCommandDisposition::Accepted,
);
let _ = cancel_tx.send(true);
}
Some(Method::Shutdown { command }) => {
if let Err(disposition) =
validate_shutdown_command(command, &shared_state)
{
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
disposition,
);
continue;
}
shutdown_after_compaction = true;
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
WorkerCommandDisposition::Accepted,
);
let _ = cancel_tx.send(true);
}
Some(method) => deferred_methods.push_back(method),
None => {
shutdown_after_compaction = true;
let _ = cancel_tx.send(true);
}
}
}
}
}
};
if !matches!(
result,
Err(WorkerError::Store(_))
| Err(WorkerError::WorkerStore(_))
| Err(WorkerError::InvalidState(_))
) {
set_controller_state(
&shared_state,
&runtime_dir,
&working_event_tx,
WorkerState::Idle,
)
.await;
}
if let Err(error) = result {
let _ = working_event_tx.send(Event::Error {
code: worker_error_code(&error),
message: error.to_string(),
});
}
if shutdown_after_compaction {
let _ = working_event_tx.send(Event::Shutdown);
break 'controller;
}
}
Method::ListRewindTargets => match shared_state.catalog_status() {
WorkerStatus::Idle | WorkerStatus::Paused => { WorkerStatus::Idle | WorkerStatus::Paused => {
emit_rewind_targets(&worker, &working_event_tx) emit_rewind_targets(&worker, &working_event_tx)
} }
@@ -1923,7 +2207,7 @@ async fn controller_loop<C, St>(
Method::RewindTo { Method::RewindTo {
target, target,
expected_head_entries, expected_head_entries,
} => match shared_state.get_status() { } => match shared_state.catalog_status() {
WorkerStatus::Idle => { WorkerStatus::Idle => {
if apply_rewind( if apply_rewind(
&mut worker, &mut worker,
@@ -1934,10 +2218,8 @@ async fn controller_loop<C, St>(
.await .await
{ {
worker.clear_in_flight_events(); worker.clear_in_flight_events();
shared_state.set_status(WorkerStatus::Idle); let snapshot = shared_state.transition(WorkerState::Idle);
let _ = working_event_tx.send(Event::Status { let _ = working_event_tx.send(Event::WorkerState { snapshot });
status: WorkerStatus::Idle,
});
} }
} }
WorkerStatus::Paused => { WorkerStatus::Paused => {
@@ -1956,7 +2238,26 @@ async fn controller_loop<C, St>(
} }
}, },
Method::Shutdown => { Method::Shutdown { command } => {
// Shutdown ignores the state-revision fence but remains bound to the
// current execution generation and command payload identity.
if let Err(disposition) = validate_shutdown_command(command, &shared_state) {
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
disposition,
);
continue;
}
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
WorkerCommandDisposition::Accepted,
);
let _ = working_event_tx.send(Event::Shutdown); let _ = working_event_tx.send(Event::Shutdown);
break; break;
} }
@@ -2038,7 +2339,7 @@ async fn controller_loop<C, St>(
// Auto-kick a turn if the Worker is idle so the // Auto-kick a turn if the Worker is idle so the
// notification is not stranded. Matches the // notification is not stranded. Matches the
// `Method::Notify` idle path. // `Method::Notify` idle path.
if shared_state.get_status() == WorkerStatus::Idle { if shared_state.catalog_status() == WorkerStatus::Idle {
pending = Some(PendingRun::RunForNotification { pending = Some(PendingRun::RunForNotification {
invoke_kind: protocol::InvokeKind::WorkerEvent, invoke_kind: protocol::InvokeKind::WorkerEvent,
notification_request_id: None, notification_request_id: None,
@@ -2294,15 +2595,115 @@ where
} }
method = method_rx.recv(), if input_commit.is_none() => { method = method_rx.recv(), if input_commit.is_none() => {
match method { match method {
Some(Method::Cancel) => { Some(Method::Cancel { command }) => {
if let Err(disposition) =
validate_command(command, WorkerCommandKind::Cancel, shared_state)
{
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Cancel,
disposition,
);
continue;
}
if !matches!(
shared_state.snapshot().state,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
) {
reject_invalid_command_state(
working_event_tx,
shared_state,
command,
WorkerCommandKind::Cancel,
);
continue;
}
set_controller_state(
shared_state,
runtime_dir,
working_event_tx,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
)
.await;
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Cancel,
WorkerCommandDisposition::Accepted,
);
let _ = cancel_tx.try_send(()); let _ = cancel_tx.try_send(());
} }
Some(Method::Pause) => { Some(Method::Pause { command }) => {
if let Err(disposition) =
validate_command(command, WorkerCommandKind::Pause, shared_state)
{
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Pause,
disposition,
);
continue;
}
if !matches!(
shared_state.snapshot().state,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
) {
reject_invalid_command_state(
working_event_tx,
shared_state,
command,
WorkerCommandKind::Pause,
);
continue;
}
pause_requested = true; pause_requested = true;
set_controller_state(
shared_state,
runtime_dir,
working_event_tx,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Pausing)),
)
.await;
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Pause,
WorkerCommandDisposition::Accepted,
);
let _ = pause_tx.try_send(()); let _ = pause_tx.try_send(());
} }
Some(Method::Shutdown) => { Some(Method::Shutdown { command }) => {
if let Err(disposition) = validate_shutdown_command(command, shared_state) {
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
disposition,
);
continue;
}
shutdown_requested = true; shutdown_requested = true;
set_controller_state(
shared_state,
runtime_dir,
working_event_tx,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
)
.await;
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
WorkerCommandDisposition::Accepted,
);
let _ = cancel_tx.try_send(()); let _ = cancel_tx.try_send(());
} }
Some(Method::Submit { Some(Method::Submit {
@@ -2368,7 +2769,27 @@ where
} }
} }
} }
Some(Method::Resume | Method::ContinuePending { .. }) => { Some(Method::Resume { command }) => {
if let Err(disposition) =
validate_command(command, WorkerCommandKind::Resume, shared_state)
{
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Resume,
disposition,
);
} else {
reject_invalid_command_state(
working_event_tx,
shared_state,
command,
WorkerCommandKind::Resume,
);
}
}
Some(Method::ContinuePending { .. }) => {
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
code: ErrorCode::AlreadyRunning, code: ErrorCode::AlreadyRunning,
message: "Worker is already executing a turn".into(), message: "Worker is already executing a turn".into(),
@@ -2408,7 +2829,27 @@ where
} }
} }
} }
Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => { Some(Method::Compact { command }) => {
if let Err(disposition) =
validate_command(command, WorkerCommandKind::Compact, shared_state)
{
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Compact,
disposition,
);
} else {
reject_invalid_command_state(
working_event_tx,
shared_state,
command,
WorkerCommandKind::Compact,
);
}
}
Some(Method::ListRewindTargets | Method::RewindTo { .. }) => {
let _ = working_event_tx.send(Event::Error { let _ = working_event_tx.send(Event::Error {
code: ErrorCode::AlreadyRunning, code: ErrorCode::AlreadyRunning,
message: "Worker is already executing a turn; rewind/compact can only run while idle or paused" message: "Worker is already executing a turn; rewind/compact can only run while idle or paused"
@@ -2511,7 +2952,7 @@ where
} }
None => { None => {
let _ = cancel_tx.try_send(()); let _ = cancel_tx.try_send(());
shared_state.set_status(WorkerStatus::Idle); shared_state.transition(WorkerState::Idle);
return (WorkerStatus::Idle, false, false); return (WorkerStatus::Idle, false, false);
} }
} }
@@ -2887,7 +3328,7 @@ mod tests {
context_window: 200_000, context_window: 200_000,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}) })
@@ -2943,9 +3384,17 @@ mod tests {
async fn pause_waits_for_run_boundary_and_uses_safe_pause_channel() { async fn pause_waits_for_run_boundary_and_uses_safe_pause_channel() {
let mut env = make_env().await; let mut env = make_env().await;
let method_tx = env._method_tx.clone(); let method_tx = env._method_tx.clone();
env.shared_state
.transition(WorkerState::Busy(WorkerBusyState::Run(
WorkerRunState::Running,
)));
let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot());
tokio::spawn(async move { tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(10)).await; tokio::time::sleep(Duration::from_millis(10)).await;
method_tx.send(Method::Pause).await.expect("send pause"); method_tx
.send(Method::Pause { command })
.await
.expect("send pause");
}); });
let worker_future = async { let worker_future = async {
@@ -3218,8 +3667,13 @@ mod tests {
async fn compact_method_is_rejected_while_running() { async fn compact_method_is_rejected_while_running() {
let mut env = make_env().await; let mut env = make_env().await;
let mut events = env.working_event_tx.subscribe(); let mut events = env.working_event_tx.subscribe();
env.shared_state
.transition(WorkerState::Busy(WorkerBusyState::Run(
WorkerRunState::Running,
)));
let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot());
env._method_tx env._method_tx
.send(Method::Compact) .send(Method::Compact { command })
.await .await
.expect("send compact"); .expect("send compact");
@@ -3252,14 +3706,113 @@ mod tests {
.expect("event timeout") .expect("event timeout")
.expect("event"); .expect("event");
match event { match event {
Event::Error { code, message } => { Event::CommandAcknowledged { acknowledgement } => {
assert_eq!(code, ErrorCode::AlreadyRunning); assert_eq!(acknowledgement.command, WorkerCommandKind::Compact);
assert!(message.contains("compact"), "got message: {message}"); assert_eq!(
acknowledgement.disposition,
WorkerCommandDisposition::InvalidState
);
assert!(matches!(
acknowledgement.state.state,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
));
} }
other => panic!("expected compact rejection error, got {other:?}"), other => panic!("expected compact rejection acknowledgement, got {other:?}"),
} }
} }
#[test]
fn command_admission_rejects_stale_generation_revision_and_order() {
let shared = WorkerSharedState::new_with_generation(
"worker".into(),
session_store::new_segment_id(),
String::new(),
protocol::Greeting {
worker_name: "worker".into(),
cwd: "/tmp".into(),
provider: "test".into(),
model: "test".into(),
scope_summary: String::new(),
tools: Vec::new(),
context_window: 1,
context_tokens: 0,
},
9,
);
assert_eq!(
validate_command(
WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 8,
expected_worker_state_revision: 0,
},
WorkerCommandKind::Pause,
&shared,
),
Err(WorkerCommandDisposition::StaleExecutionGeneration)
);
assert_eq!(
validate_command(
WorkerCommandEnvelope {
command_id: 2,
expected_execution_generation: 9,
expected_worker_state_revision: 1,
},
WorkerCommandKind::Pause,
&shared,
),
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
);
assert!(
validate_command(
WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 9,
expected_worker_state_revision: 0,
},
WorkerCommandKind::Pause,
&shared,
)
.is_ok()
);
assert_eq!(
validate_command(
WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 9,
expected_worker_state_revision: 0,
},
WorkerCommandKind::Pause,
&shared,
),
Err(WorkerCommandDisposition::StaleCommandId)
);
assert_eq!(
validate_command(
WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 9,
expected_worker_state_revision: 0,
},
WorkerCommandKind::Cancel,
&shared,
),
Err(WorkerCommandDisposition::Conflict)
);
assert!(
validate_command(
WorkerCommandEnvelope {
command_id: 2,
expected_execution_generation: 9,
expected_worker_state_revision: 1,
},
WorkerCommandKind::Pause,
&shared,
)
.is_ok()
);
}
#[test] #[test]
fn controller_shutdown_orders_child_cleanup_before_workdir_close() { fn controller_shutdown_orders_child_cleanup_before_workdir_close() {
let source = include_str!("controller.rs"); let source = include_str!("controller.rs");
+8 -8
View File
@@ -779,10 +779,10 @@ async fn probe_socket(socket_path: &Path) -> LiveInfo {
loop { loop {
match tokio::time::timeout(PROBE_TIMEOUT, reader.next::<Event>()).await { match tokio::time::timeout(PROBE_TIMEOUT, reader.next::<Event>()).await {
Ok(Ok(Some(Event::Snapshot { Ok(Ok(Some(Event::Snapshot {
status: snapshot_status, state: snapshot_state,
.. ..
}))) => { }))) => {
status = Some(snapshot_status); status = Some(snapshot_state.catalog_status());
break; break;
} }
Ok(Ok(Some(Event::Alert(_)))) => continue, Ok(Ok(Some(Event::Alert(_)))) => continue,
@@ -1507,7 +1507,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}) })
@@ -1543,7 +1543,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}) })
@@ -1638,7 +1638,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}) })
@@ -1665,7 +1665,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}) })
@@ -1773,7 +1773,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Paused, state: WorkerStatus::Paused.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}) })
@@ -1827,7 +1827,7 @@ mod tests {
context_window: 0, context_window: 0,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
}) })
+51 -19
View File
@@ -295,6 +295,38 @@ impl InternalWorkerSessionStatus {
} }
} }
fn send_internal_worker_state(
event_tx: &broadcast::Sender<Event>,
state_revision: &std::sync::atomic::AtomicU64,
status: InternalWorkerSessionStatus,
) {
let state = match status {
InternalWorkerSessionStatus::Idle
| InternalWorkerSessionStatus::Stopped
| InternalWorkerSessionStatus::Failed => protocol::WorkerState::Idle,
InternalWorkerSessionStatus::Paused => protocol::WorkerState::Busy(
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Paused),
),
InternalWorkerSessionStatus::Running => protocol::WorkerState::Busy(
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running),
),
InternalWorkerSessionStatus::Stopping => protocol::WorkerState::Busy(
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Cancelling),
),
};
let revision = state_revision
.fetch_add(1, std::sync::atomic::Ordering::AcqRel)
.saturating_add(1);
let _ = event_tx.send(Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
execution_generation: 1,
revision,
last_command_id: 0,
state,
},
});
}
fn classify_internal_turn_result( fn classify_internal_turn_result(
result: Result<WorkerRunResult, WorkerError>, result: Result<WorkerRunResult, WorkerError>,
) -> (InternalWorkerSessionStatus, Option<String>) { ) -> (InternalWorkerSessionStatus, Option<String>) {
@@ -351,6 +383,7 @@ pub(crate) struct InternalWorkerSessionSnapshot {
pub(crate) struct InternalWorkerSessionHandle { pub(crate) struct InternalWorkerSessionHandle {
command_tx: tokio::sync::mpsc::Sender<InternalWorkerSessionCommand>, command_tx: tokio::sync::mpsc::Sender<InternalWorkerSessionCommand>,
status: Arc<std::sync::atomic::AtomicU8>, status: Arc<std::sync::atomic::AtomicU8>,
state_revision: Arc<std::sync::atomic::AtomicU64>,
store: EphemeralSessionStore, store: EphemeralSessionStore,
session_id: SessionId, session_id: SessionId,
segment_id: SegmentId, segment_id: SegmentId,
@@ -400,6 +433,10 @@ impl InternalWorkerSessionHandle {
self.in_flight.text_delta(block_id, text.to_owned()); self.in_flight.text_delta(block_id, text.to_owned());
} }
fn emit_worker_state(&self, status: InternalWorkerSessionStatus) {
send_internal_worker_state(&self.event_tx, &self.state_revision, status);
}
pub(crate) fn protocol_snapshot(&self) -> InternalWorkerSessionSnapshot { pub(crate) fn protocol_snapshot(&self) -> InternalWorkerSessionSnapshot {
let (entries, in_flight) = { let (entries, in_flight) = {
let guard = self.in_flight.snapshot_guard(); let guard = self.in_flight.snapshot_guard();
@@ -473,9 +510,7 @@ impl InternalWorkerSessionHandle {
}); });
return Err(InternalWorkerSessionError::Unavailable); return Err(InternalWorkerSessionError::Unavailable);
} }
let _ = self.event_tx.send(Event::Status { self.emit_worker_state(InternalWorkerSessionStatus::Running);
status: WorkerStatus::Running,
});
Ok(()) Ok(())
} }
@@ -770,11 +805,13 @@ pub(crate) async fn prepare_internal_worker_session(
let status = Arc::new(std::sync::atomic::AtomicU8::new( let status = Arc::new(std::sync::atomic::AtomicU8::new(
InternalWorkerSessionStatus::Idle.encode(), InternalWorkerSessionStatus::Idle.encode(),
)); ));
let state_revision = Arc::new(std::sync::atomic::AtomicU64::new(0));
let state_changed = Arc::new(tokio::sync::Notify::new()); let state_changed = Arc::new(tokio::sync::Notify::new());
let last_error = Arc::new(Mutex::new(None)); let last_error = Arc::new(Mutex::new(None));
let handle = InternalWorkerSessionHandle { let handle = InternalWorkerSessionHandle {
command_tx, command_tx,
status: status.clone(), status: status.clone(),
state_revision: state_revision.clone(),
store, store,
session_id, session_id,
segment_id, segment_id,
@@ -810,19 +847,11 @@ pub(crate) async fn prepare_internal_worker_session(
message, message,
}); });
} }
let protocol_status = match turn_status { send_internal_worker_state(
InternalWorkerSessionStatus::Idle => WorkerStatus::Idle, &event_tx,
InternalWorkerSessionStatus::Paused => WorkerStatus::Paused, &state_revision,
InternalWorkerSessionStatus::Stopped turn_status,
| InternalWorkerSessionStatus::Failed => WorkerStatus::Stopped, );
InternalWorkerSessionStatus::Running
| InternalWorkerSessionStatus::Stopping => {
unreachable!("run completion cannot remain active")
}
};
let _ = event_tx.send(Event::Status {
status: protocol_status,
});
if let Some(callback) = &on_turn_end { if let Some(callback) = &on_turn_end {
callback(turn_status); callback(turn_status);
} }
@@ -864,9 +893,11 @@ pub(crate) async fn prepare_internal_worker_session(
InternalWorkerSessionStatus::Stopped.encode(), InternalWorkerSessionStatus::Stopped.encode(),
std::sync::atomic::Ordering::Release, std::sync::atomic::Ordering::Release,
); );
let _ = event_tx.send(Event::Status { send_internal_worker_state(
status: WorkerStatus::Stopped, &event_tx,
}); &state_revision,
InternalWorkerSessionStatus::Stopped,
);
let _ = event_tx.send(Event::Shutdown); let _ = event_tx.send(Event::Shutdown);
state_changed.notify_waiters(); state_changed.notify_waiters();
if let Some(done) = stop_done { if let Some(done) = stop_done {
@@ -1118,6 +1149,7 @@ pub(crate) fn test_internal_worker_session(
status: Arc::new(std::sync::atomic::AtomicU8::new( status: Arc::new(std::sync::atomic::AtomicU8::new(
InternalWorkerSessionStatus::Idle.encode(), InternalWorkerSessionStatus::Idle.encode(),
)), )),
state_revision: Arc::new(std::sync::atomic::AtomicU64::new(0)),
store, store,
session_id, session_id,
segment_id, segment_id,
+3 -2
View File
@@ -197,7 +197,6 @@ pub fn default_base() -> Result<PathBuf, io::Error> {
mod tests { mod tests {
use super::*; use super::*;
use crate::shared_state::WorkerSharedState; use crate::shared_state::WorkerSharedState;
use protocol::WorkerStatus;
fn test_state() -> WorkerSharedState { fn test_state() -> WorkerSharedState {
WorkerSharedState::new( WorkerSharedState::new(
@@ -247,7 +246,9 @@ mod tests {
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
let state = test_state(); let state = test_state();
state.set_status(WorkerStatus::Running); state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)));
rt.write_status(&state).await.unwrap(); rt.write_status(&state).await.unwrap();
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap(); let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
+240 -42
View File
@@ -1,28 +1,49 @@
use std::sync::atomic::{AtomicBool, Ordering}; use std::collections::VecDeque;
use std::sync::{OnceLock, RwLock}; use std::sync::{
OnceLock, RwLock,
atomic::{AtomicBool, Ordering},
};
use protocol::WorkerStatus; use protocol::{
WorkerBusyState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind,
WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus,
};
use serde_json::json; use serde_json::json;
use session_store::SegmentId; use session_store::SegmentId;
use crate::fs_view::WorkerFsView; use crate::fs_view::WorkerFsView;
const COMPLETED_COMMAND_RETENTION: usize = 256;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct AcceptedWorkerCommand {
envelope: WorkerCommandEnvelope,
kind: WorkerCommandKind,
disposition: Option<WorkerCommandDisposition>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum WorkerCommandAdmission {
Accepted,
Retry,
Conflict,
StaleCommandId,
ExecutionGenerationMismatch,
StateRevisionMismatch,
}
/// Shared state between WorkerController and runtime directory. /// Shared state between WorkerController and runtime directory.
/// ///
/// Controller updates this in-memory; RuntimeDir writes the status /// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime
/// snapshot to disk. Wrapped in `Arc` for sharing. /// catalog status remains a separate lifecycle projection because `Stopped`
/// /// describes the execution handle rather than a live controller state.
/// History and typed user-segment mirrors used to live here so the
/// IPC layer could answer `Method::GetHistory`. Those reads now go
/// directly through the session-log sink (`Event::Snapshot` +
/// live events), so this struct holds only status, identity,
/// greeting, and filesystem completion lookup hubs.
pub struct WorkerSharedState { pub struct WorkerSharedState {
pub worker_name: String, pub worker_name: String,
pub segment_id: SegmentId, pub segment_id: SegmentId,
pub manifest_toml: String, pub manifest_toml: String,
pub greeting: protocol::Greeting, pub greeting: protocol::Greeting,
pub status: RwLock<WorkerStatus>, state: RwLock<WorkerStateSnapshot>,
accepted_commands: RwLock<VecDeque<AcceptedWorkerCommand>>,
/// Worker-from-the-inside view of the filesystem. Set once in /// Worker-from-the-inside view of the filesystem. Set once in
/// `WorkerController::start` after the local WorkdirSession provider is /// `WorkerController::start` after the local WorkdirSession provider is
/// materialised, and read from the IPC server layer to answer /// materialised, and read from the IPC server layer to answer
@@ -38,13 +59,24 @@ impl WorkerSharedState {
segment_id: SegmentId, segment_id: SegmentId,
manifest_toml: String, manifest_toml: String,
greeting: protocol::Greeting, greeting: protocol::Greeting,
) -> Self {
Self::new_with_generation(worker_name, segment_id, manifest_toml, greeting, 1)
}
pub fn new_with_generation(
worker_name: String,
segment_id: SegmentId,
manifest_toml: String,
greeting: protocol::Greeting,
execution_generation: u64,
) -> Self { ) -> Self {
Self { Self {
worker_name, worker_name,
segment_id, segment_id,
manifest_toml, manifest_toml,
greeting, greeting,
status: RwLock::new(WorkerStatus::Idle), state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)),
accepted_commands: RwLock::new(VecDeque::new()),
fs_view: OnceLock::new(), fs_view: OnceLock::new(),
flow_transition_enabled: AtomicBool::new(false), flow_transition_enabled: AtomicBool::new(false),
} }
@@ -70,21 +102,140 @@ impl WorkerSharedState {
self.flow_transition_enabled.load(Ordering::Acquire) self.flow_transition_enabled.load(Ordering::Acquire)
} }
pub fn set_status(&self, status: WorkerStatus) { pub fn transition(&self, state: WorkerState) -> WorkerStateSnapshot {
if let Ok(mut s) = self.status.write() { let mut snapshot = self
*s = status; .state
.write()
.expect("worker state lock poisoned; refusing an inferred fallback state");
if snapshot.state != state {
snapshot.revision = snapshot.revision.saturating_add(1);
snapshot.state = state;
}
snapshot.clone()
}
pub(crate) fn admit_command(
&self,
envelope: WorkerCommandEnvelope,
kind: WorkerCommandKind,
require_state_revision: bool,
) -> WorkerCommandAdmission {
let mut snapshot = self
.state
.write()
.expect("worker state lock poisoned; refusing command admission");
let mut accepted = self
.accepted_commands
.write()
.expect("worker command ledger lock poisoned; refusing command admission");
if let Some(existing) = accepted
.iter()
.find(|accepted| accepted.envelope.command_id == envelope.command_id)
{
return if existing.envelope == envelope && existing.kind == kind {
WorkerCommandAdmission::Retry
} else {
WorkerCommandAdmission::Conflict
};
}
if envelope.expected_execution_generation != snapshot.execution_generation {
return WorkerCommandAdmission::ExecutionGenerationMismatch;
}
if require_state_revision && envelope.expected_worker_state_revision != snapshot.revision {
return WorkerCommandAdmission::StateRevisionMismatch;
}
if envelope.command_id <= snapshot.last_command_id {
return WorkerCommandAdmission::StaleCommandId;
}
snapshot.last_command_id = envelope.command_id;
snapshot.revision = snapshot.revision.saturating_add(1);
accepted.push_back(AcceptedWorkerCommand {
envelope,
kind,
disposition: None,
});
WorkerCommandAdmission::Accepted
}
pub(crate) fn complete_command(
&self,
command_id: u64,
kind: WorkerCommandKind,
disposition: WorkerCommandDisposition,
) {
if !matches!(
disposition,
WorkerCommandDisposition::Accepted | WorkerCommandDisposition::InvalidState
) {
return;
}
let mut accepted = self
.accepted_commands
.write()
.expect("worker command ledger lock poisoned; refusing command completion");
if let Some(command) = accepted
.iter_mut()
.find(|command| command.envelope.command_id == command_id && command.kind == kind)
{
command.disposition.get_or_insert(disposition);
}
while accepted
.iter()
.filter(|command| command.disposition.is_some())
.count()
> COMPLETED_COMMAND_RETENTION
{
let Some(index) = accepted
.iter()
.position(|command| command.disposition.is_some())
else {
break;
};
accepted.remove(index);
} }
} }
pub fn get_status(&self) -> WorkerStatus { #[cfg(test)]
self.status.read().map(|s| *s).unwrap_or(WorkerStatus::Idle) pub(crate) fn command_result(
&self,
command_id: u64,
) -> Option<Option<WorkerCommandDisposition>> {
self.accepted_commands
.read()
.expect("worker command ledger lock poisoned")
.iter()
.find(|command| command.envelope.command_id == command_id)
.map(|command| command.disposition)
} }
/// Serialize status as JSON. pub fn snapshot(&self) -> WorkerStateSnapshot {
self.state
.read()
.expect("worker state lock poisoned; refusing an inferred fallback state")
.clone()
}
/// Runtime catalog projection. This must not be used as live command
/// admission authority.
pub fn catalog_status(&self) -> WorkerStatus {
match self.snapshot().state {
WorkerState::Idle => WorkerStatus::Idle,
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused,
WorkerState::Busy(WorkerBusyState::Run(_))
| WorkerState::Busy(WorkerBusyState::Maintenance(WorkerMaintenanceState::Compacting)) => {
WorkerStatus::Running
}
}
}
/// Serialize the runtime-directory lifecycle projection as JSON while
/// retaining the full state snapshot for diagnostics and reconnects.
pub fn status_json(&self) -> String { pub fn status_json(&self) -> String {
let status = self.get_status(); let snapshot = self.snapshot();
json!({ json!({
"state": status, "state": self.catalog_status(),
"worker_state": snapshot,
"segment_id": self.segment_id.to_string(), "segment_id": self.segment_id.to_string(),
"worker_name": self.worker_name, "worker_name": self.worker_name,
}) })
@@ -97,11 +248,12 @@ mod tests {
use super::*; use super::*;
fn test_state() -> WorkerSharedState { fn test_state() -> WorkerSharedState {
WorkerSharedState::new( WorkerSharedState::new_with_generation(
"test-worker".into(), "test-worker".into(),
session_store::new_segment_id(), session_store::new_segment_id(),
"[engine]\nname = \"test-worker\"".into(), "[engine]\nname = \"test-worker\"".into(),
test_greeting(), test_greeting(),
7,
) )
} }
@@ -119,36 +271,82 @@ mod tests {
} }
#[test] #[test]
fn initial_status_is_idle() { fn initial_snapshot_is_idle() {
let state = test_state(); let state = test_state();
assert_eq!(state.get_status(), WorkerStatus::Idle); assert_eq!(state.snapshot(), WorkerStateSnapshot::initial(7));
assert_eq!(state.catalog_status(), WorkerStatus::Idle);
} }
#[test] #[test]
fn set_and_get_status() { fn transitions_increment_revision_only_when_state_changes() {
let state = test_state(); let state = test_state();
state.set_status(WorkerStatus::Running); let running = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running));
assert_eq!(state.get_status(), WorkerStatus::Running); let snapshot = state.transition(running.clone());
state.set_status(WorkerStatus::Paused); assert_eq!(snapshot.revision, 1);
assert_eq!(state.get_status(), WorkerStatus::Paused); assert_eq!(snapshot.state, running);
assert_eq!(state.transition(running).revision, 1);
let paused = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused));
let snapshot = state.transition(paused.clone());
assert_eq!(snapshot.revision, 2);
assert_eq!(snapshot.state, paused);
assert_eq!(state.catalog_status(), WorkerStatus::Paused);
} }
#[test] #[test]
fn status_json_contains_fields() { fn accepted_command_identity_advances_revision_and_detects_reuse_conflicts() {
let state = test_state(); let state = test_state();
let json = state.status_json(); let envelope = WorkerCommandEnvelope {
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); command_id: 9,
assert_eq!(parsed["state"], "idle"); expected_execution_generation: 7,
expected_worker_state_revision: 0,
};
assert_eq!(
state.admit_command(envelope, WorkerCommandKind::Pause, true),
WorkerCommandAdmission::Accepted
);
assert_eq!(
state.snapshot(),
WorkerStateSnapshot {
execution_generation: 7,
revision: 1,
last_command_id: 9,
state: WorkerState::Idle,
}
);
assert_eq!(state.command_result(9), Some(None));
state.complete_command(
9,
WorkerCommandKind::Pause,
WorkerCommandDisposition::Accepted,
);
assert_eq!(
state.command_result(9),
Some(Some(WorkerCommandDisposition::Accepted))
);
assert_eq!(
state.admit_command(envelope, WorkerCommandKind::Pause, true),
WorkerCommandAdmission::Retry
);
assert_eq!(
state.admit_command(envelope, WorkerCommandKind::Cancel, true),
WorkerCommandAdmission::Conflict
);
assert_eq!(state.snapshot().revision, 1);
}
#[test]
fn status_json_contains_full_snapshot_and_catalog_projection() {
let state = test_state();
state.transition(WorkerState::Busy(WorkerBusyState::Maintenance(
WorkerMaintenanceState::Compacting,
)));
let parsed: serde_json::Value = serde_json::from_str(&state.status_json()).unwrap();
assert_eq!(parsed["state"], "running");
assert_eq!(parsed["worker_state"]["execution_generation"], 7);
assert_eq!(parsed["worker_state"]["revision"], 1);
assert_eq!(parsed["worker_state"]["state"]["kind"], "busy");
assert_eq!(parsed["worker_name"], "test-worker"); assert_eq!(parsed["worker_name"], "test-worker");
assert!(parsed["segment_id"].is_string()); assert!(parsed["segment_id"].is_string());
} }
#[test]
fn status_json_reflects_changes() {
let state = test_state();
state.set_status(WorkerStatus::Running);
let json = state.status_json();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["state"], "running");
}
} }
+9 -3
View File
@@ -97,7 +97,7 @@ mod tests {
context_window: 200_000, context_window: 200_000,
context_tokens: 0, context_tokens: 0,
}, },
status: WorkerStatus::Idle, state: WorkerStatus::Idle.into(),
in_flight: Default::default(), in_flight: Default::default(),
internal_workers: Vec::new(), internal_workers: Vec::new(),
} }
@@ -137,10 +137,16 @@ mod tests {
], ],
); );
connect_and_send(&socket, &Method::Shutdown).await.unwrap(); let method = Method::Shutdown {
command: protocol::WorkerCommandEnvelope::for_snapshot(
1,
&protocol::WorkerStateSnapshot::initial(1),
),
};
connect_and_send(&socket, &method).await.unwrap();
let method = received.await.unwrap().expect("expected method"); let method = received.await.unwrap().expect("expected method");
assert!(matches!(method, Method::Shutdown)); assert!(matches!(method, Method::Shutdown { .. }));
} }
#[tokio::test] #[tokio::test]
+186 -20
View File
@@ -4571,15 +4571,6 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
Ok(()) Ok(())
} }
fn persist_and_send_compact_done(
&mut self,
lifecycle: CompactionLifecycle,
) -> Result<(), WorkerError> {
self.persist_compaction_lifecycle(&lifecycle)?;
self.send_event(Event::CompactDone { lifecycle });
Ok(())
}
fn persist_and_send_compact_failed( fn persist_and_send_compact_failed(
&mut self, &mut self,
lifecycle: CompactionLifecycle, lifecycle: CompactionLifecycle,
@@ -4724,7 +4715,97 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
Ok(rewrite_guard) Ok(rewrite_guard)
} }
/// Terminalize and clean up any compaction that was left active by the
/// previous controller generation. This runs before the restored
/// controller publishes its first Idle state.
pub async fn recover_unfinished_compaction(&mut self) -> Result<(), WorkerError> {
let (entries, _) = self.sink.subscribe_with_snapshot();
let latest_payload = entries.iter().rev().find_map(|entry| match entry {
LogEntry::Extension {
domain, payload, ..
} if domain == COMPACTION_EXTENSION_DOMAIN => Some(payload.clone()),
_ => None,
});
let Some(payload) = latest_payload else {
return Ok(());
};
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct CompactionLifecycleWire {
schema_version: u32,
compaction_id: String,
revision: u64,
#[serde(default)]
internal_worker: Option<protocol::InternalWorkerRef>,
state: CompactionLifecycleState,
started_at_ms: u64,
#[serde(default)]
ended_at_ms: Option<u64>,
#[serde(default)]
summary: Option<String>,
#[serde(default)]
error: Option<String>,
#[serde(default)]
new_segment_id: Option<String>,
}
let wire: CompactionLifecycleWire = serde_json::from_value(payload).map_err(|error| {
WorkerError::InvalidState(format!("decode compaction lifecycle: {error}"))
})?;
if !matches!(wire.schema_version, 2 | 3) {
return Err(WorkerError::InvalidState(format!(
"unsupported compaction lifecycle schema version {}",
wire.schema_version
)));
}
let mut lifecycle = CompactionLifecycle {
schema_version: wire.schema_version,
compaction_id: wire.compaction_id,
revision: wire.revision,
internal_worker: wire.internal_worker,
state: wire.state,
started_at_ms: wire.started_at_ms,
ended_at_ms: wire.ended_at_ms,
summary: wire.summary,
error: wire.error,
new_segment_id: wire.new_segment_id,
};
match lifecycle.state {
CompactionLifecycleState::Running => {
lifecycle.schema_version = 3;
lifecycle.revision = lifecycle.revision.saturating_add(1);
lifecycle.state = CompactionLifecycleState::Interrupted;
lifecycle.ended_at_ms = Some(segment_log::now_millis());
lifecycle.error =
Some("worker execution restarted before compaction completed".into());
self.persist_compaction_lifecycle(&lifecycle)?;
self.send_event(Event::CompactFailed {
lifecycle: lifecycle.clone(),
});
self.release_compaction_service(&lifecycle).await;
}
CompactionLifecycleState::Interrupted => {
self.release_compaction_service(&lifecycle).await;
}
CompactionLifecycleState::Done | CompactionLifecycleState::Failed => {}
}
Ok(())
}
pub async fn manual_compact(&mut self) -> Result<ManualCompactResult, WorkerError> { pub async fn manual_compact(&mut self) -> Result<ManualCompactResult, WorkerError> {
self.manual_compact_inner(None).await
}
pub async fn manual_compact_with_cancel(
&mut self,
cancel: tokio::sync::watch::Receiver<bool>,
) -> Result<ManualCompactResult, WorkerError> {
self.manual_compact_inner(Some(cancel)).await
}
async fn manual_compact_inner(
&mut self,
mut cancel: Option<tokio::sync::watch::Receiver<bool>>,
) -> Result<ManualCompactResult, WorkerError> {
if self.manifest.compaction.is_none() { if self.manifest.compaction.is_none() {
let message = let message =
"manual compact is unavailable because [compaction] is not configured".to_string(); "manual compact is unavailable because [compaction] is not configured".to_string();
@@ -4764,7 +4845,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
return Ok(ManualCompactResult::Skipped { message }); return Ok(ManualCompactResult::Skipped { message });
} }
match self.compact(retained).await { match self.compact_with_cancel(retained, cancel.take()).await {
Ok(new_segment_id) => { Ok(new_segment_id) => {
info!(new_segment_id = %new_segment_id, "Manual compaction succeeded"); info!(new_segment_id = %new_segment_id, "Manual compaction succeeded");
if let Some(ref state) = state { if let Some(ref state) = state {
@@ -4937,11 +5018,19 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
/// Runs one parent-owned observable compaction service and returns the new /// Runs one parent-owned observable compaction service and returns the new
/// Segment ID. Lifecycle revisions are committed before they are broadcast. /// Segment ID. Lifecycle revisions are committed before they are broadcast.
pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> { pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> {
self.compact_with_cancel(retained_tokens, None).await
}
async fn compact_with_cancel(
&mut self,
retained_tokens: u64,
mut cancel: Option<tokio::sync::watch::Receiver<bool>>,
) -> Result<SegmentId, WorkerError> {
let _rewrite_guard = self let _rewrite_guard = self
.prepare_session_rewrite(SessionRewriteKind::Compact) .prepare_session_rewrite(SessionRewriteKind::Compact)
.await?; .await?;
let mut lifecycle = CompactionLifecycle { let mut lifecycle = CompactionLifecycle {
schema_version: 2, schema_version: 3,
compaction_id: uuid::Uuid::now_v7().to_string(), compaction_id: uuid::Uuid::now_v7().to_string(),
revision: 1, revision: 1,
internal_worker: None, internal_worker: None,
@@ -4953,16 +5042,25 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
new_segment_id: None, new_segment_id: None,
}; };
self.persist_and_send_compact_start(lifecycle.clone())?; self.persist_and_send_compact_start(lifecycle.clone())?;
match self.compact_impl(retained_tokens, &mut lifecycle).await { let outcome = if let Some(cancel) = cancel.as_mut() {
Ok((new_segment_id, summary)) => { tokio::select! {
lifecycle.revision = lifecycle.revision.saturating_add(1); biased;
lifecycle.state = CompactionLifecycleState::Done; changed = cancel.changed() => {
lifecycle.ended_at_ms = Some(segment_log::now_millis()); let _ = changed;
lifecycle.summary = Some(summary); Err(WorkerError::CompactCancelled)
lifecycle.new_segment_id = Some(new_segment_id.to_string()); }
let terminal = self.persist_and_send_compact_done(lifecycle.clone()); result = self.compact_impl(retained_tokens, &mut lifecycle) => result,
}
} else {
self.compact_impl(retained_tokens, &mut lifecycle).await
};
match outcome {
Ok((new_segment_id, _summary)) => {
debug_assert_eq!(lifecycle.state, CompactionLifecycleState::Done);
self.send_event(Event::CompactDone {
lifecycle: lifecycle.clone(),
});
self.release_compaction_service(&lifecycle).await; self.release_compaction_service(&lifecycle).await;
terminal?;
Ok(new_segment_id) Ok(new_segment_id)
} }
Err(error) => { Err(error) => {
@@ -5543,6 +5641,24 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
})?, })?,
}); });
} }
// Commit the terminal lifecycle in the same atomic replacement-segment
// creation as the rewritten history. Restore can therefore never see a
// replacement segment without the Done fact for the compaction that
// created it.
lifecycle.revision = lifecycle.revision.saturating_add(1);
lifecycle.state = CompactionLifecycleState::Done;
lifecycle.ended_at_ms = Some(segment_log::now_millis());
lifecycle.summary = Some(summary_text.clone());
lifecycle.new_segment_id = Some(new_segment_id.to_string());
initial_entries.push(LogEntry::Extension {
ts: segment_log::now_millis(),
domain: COMPACTION_EXTENSION_DOMAIN.to_string(),
payload: serde_json::to_value(&*lifecycle).map_err(|error| {
WorkerError::InvalidState(format!(
"serialize terminal compaction lifecycle: {error}"
))
})?,
});
self.store self.store
.create_segment(old_loc.session_id, new_segment_id, &initial_entries)?; .create_segment(old_loc.session_id, new_segment_id, &initial_entries)?;
self.segment_state.set_location(SegmentLocation { self.segment_state.set_location(SegmentLocation {
@@ -10165,6 +10281,56 @@ mod build_summary_prompt_tests {
assert_eq!(state.notification_receipts.len(), 1); assert_eq!(state.notification_receipts.len(), 1);
} }
#[tokio::test]
async fn restore_terminalizes_running_compaction_before_idle_publication() {
let (_dir, mut worker) = rewind_test_worker().await;
let lifecycle = CompactionLifecycle {
schema_version: 3,
compaction_id: "compact-before-restart".into(),
revision: 1,
internal_worker: None,
state: CompactionLifecycleState::Running,
started_at_ms: segment_log::now_millis(),
ended_at_ms: None,
summary: None,
error: None,
new_segment_id: None,
};
worker.persist_compaction_lifecycle(&lifecycle).unwrap();
worker.recover_unfinished_compaction().await.unwrap();
let (entries, _) = worker.sink.subscribe_with_snapshot();
let restored = entries.iter().rev().find_map(|entry| match entry {
LogEntry::Extension {
domain, payload, ..
} if domain == COMPACTION_EXTENSION_DOMAIN => {
serde_json::from_value::<CompactionLifecycle>(payload.clone()).ok()
}
_ => None,
});
let restored = restored.expect("terminal compaction lifecycle");
assert_eq!(restored.state, CompactionLifecycleState::Interrupted);
assert_eq!(restored.revision, 2);
assert!(
restored
.error
.as_deref()
.is_some_and(|error| error.contains("restarted"))
);
let mut future = lifecycle;
future.schema_version = 4;
future.compaction_id = "future-compaction".into();
worker.persist_compaction_lifecycle(&future).unwrap();
let error = worker.recover_unfinished_compaction().await.unwrap_err();
assert!(
error
.to_string()
.contains("unsupported compaction lifecycle schema version 4")
);
}
fn minimal_manifest() -> WorkerManifest { fn minimal_manifest() -> WorkerManifest {
let toml_str = r#" let toml_str = r#"
[worker] [worker]
+202 -6
View File
@@ -72,6 +72,41 @@ impl LlmClient for MockClient {
} }
} }
#[derive(Clone)]
struct BlockingCompactClient {
calls: Arc<AtomicUsize>,
}
impl BlockingCompactClient {
fn new() -> Self {
Self {
calls: Arc::new(AtomicUsize::new(0)),
}
}
}
#[async_trait]
impl LlmClient for BlockingCompactClient {
fn clone_boxed(&self) -> Box<dyn LlmClient> {
Box::new(self.clone())
}
async fn stream(
&self,
_request: Request,
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
{
let call = self.calls.fetch_add(1, Ordering::SeqCst);
if call == 0 {
Ok(Box::pin(futures::stream::iter(
single_text_events("seed").into_iter().map(Ok),
)))
} else {
Ok(Box::pin(futures::stream::pending()))
}
}
}
fn single_text_events(text: &str) -> Vec<LlmEvent> { fn single_text_events(text: &str) -> Vec<LlmEvent> {
vec![ vec![
LlmEvent::text_block_start(0), LlmEvent::text_block_start(0),
@@ -156,10 +191,10 @@ target = "./"
permission = "write" permission = "write"
"#; "#;
async fn make_worker_with_manifest( async fn make_worker_with_manifest<C>(manifest_toml: &str, client: C) -> Worker<C, TestStore>
manifest_toml: &str, where
client: MockClient, C: LlmClient + Clone + Send + Sync + 'static,
) -> Worker<MockClient, TestStore> { {
let manifest = worker::WorkerManifest::from_toml(manifest_toml).unwrap(); let manifest = worker::WorkerManifest::from_toml(manifest_toml).unwrap();
let store_tmp = tempfile::tempdir().unwrap(); let store_tmp = tempfile::tempdir().unwrap();
@@ -614,12 +649,144 @@ async fn pre_run_compact_failure_broadcasts_start_and_failed() {
); );
} }
#[tokio::test]
async fn manual_compact_cancel_terminalizes_before_returning_idle() {
let worker =
make_worker_with_manifest(POST_RUN_MANIFEST_TOML, BlockingCompactClient::new()).await;
let runtime_tmp = tempfile::tempdir().unwrap();
let bash_output_dir = runtime_tmp.path().join("bash-output");
let (handle, shutdown_receiver) =
WorkerController::spawn(worker, runtime_tmp.path(), &bash_output_dir)
.await
.unwrap();
let mut rx = handle.subscribe();
handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"seed history",
))
.await
.expect("send seed run");
loop {
if matches!(
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
.await
.expect("timeout waiting for seed run")
.expect("event"),
Event::RunEnd {
result: RunResult::Finished
}
) {
break;
}
}
let compact = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot());
handle
.send(Method::Compact { command: compact })
.await
.expect("send compact");
loop {
if matches!(
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
.await
.expect("timeout waiting for compact start")
.expect("event"),
Event::CompactStart { .. }
) {
break;
}
}
let cancel = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot());
handle
.send(Method::Cancel { command: cancel })
.await
.expect("send compact cancel");
let mut saw_interrupted = false;
let mut saw_idle = false;
while !(saw_interrupted && saw_idle) {
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
.await
.expect("timeout waiting for compact cancellation")
.expect("event")
{
Event::CompactFailed { lifecycle }
if lifecycle.state == protocol::CompactionLifecycleState::Interrupted =>
{
saw_interrupted = true;
}
Event::WorkerState { snapshot }
if snapshot.catalog_status() == protocol::WorkerStatus::Idle =>
{
assert!(
saw_interrupted,
"Idle must follow durable Interrupted evidence"
);
saw_idle = true;
}
_ => {}
}
}
let compact = protocol::WorkerCommandEnvelope::for_snapshot(3, &handle.shared_state.snapshot());
handle
.send(Method::Compact { command: compact })
.await
.expect("send second compact");
loop {
if matches!(
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
.await
.expect("timeout waiting for second compact start")
.expect("event"),
Event::CompactStart { .. }
) {
break;
}
}
let shutdown =
protocol::WorkerCommandEnvelope::for_snapshot(4, &handle.shared_state.snapshot());
handle
.send(Method::Shutdown { command: shutdown })
.await
.expect("send shutdown during compact");
let mut interrupted_before_shutdown = false;
loop {
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
.await
.expect("timeout waiting for shutdown")
.expect("event")
{
Event::CompactFailed { lifecycle }
if lifecycle.state == protocol::CompactionLifecycleState::Interrupted =>
{
interrupted_before_shutdown = true;
}
Event::Shutdown => {
assert!(
interrupted_before_shutdown,
"shutdown must await terminal compaction evidence"
);
break;
}
_ => {}
}
}
tokio::time::timeout(std::time::Duration::from_secs(2), shutdown_receiver)
.await
.expect("controller shutdown timeout")
.expect("shutdown confirmation");
}
#[tokio::test] #[tokio::test]
async fn controller_compact_method_emits_start_and_done() { async fn controller_compact_method_emits_start_and_done() {
let client = MockClient::new(vec![ let client = MockClient::new(vec![
text_events_with_usage("hi", 1000), text_events_with_usage("hi", 1000),
write_summary_tool_use_events("manual-summary", "manual compact summary"), write_summary_tool_use_events("manual-summary", "manual compact summary"),
single_text_events("done"), single_text_events("done"),
single_text_events("follow-up"),
]); ]);
let worker = make_worker_with_manifest(POST_RUN_MANIFEST_TOML, client).await; let worker = make_worker_with_manifest(POST_RUN_MANIFEST_TOML, client).await;
let runtime_tmp = tempfile::tempdir().unwrap(); let runtime_tmp = tempfile::tempdir().unwrap();
@@ -649,7 +816,11 @@ async fn controller_compact_method_emits_start_and_done() {
} }
} }
handle.send(Method::Compact).await.expect("send compact"); let command = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot());
handle
.send(Method::Compact { command })
.await
.expect("send compact");
let mut saw_start = false; let mut saw_start = false;
loop { loop {
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
@@ -670,5 +841,30 @@ async fn controller_compact_method_emits_start_and_done() {
} }
assert!(saw_start, "manual compact should emit CompactStart"); assert!(saw_start, "manual compact should emit CompactStart");
let _ = handle.send(Method::Shutdown).await; handle
.send(Method::submit_text(
protocol::new_submission_request_id(),
"run after compact",
))
.await
.expect("send follow-up run");
loop {
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
.await
.expect("timeout waiting for follow-up run")
.expect("event")
{
Event::RunEnd {
result: RunResult::Finished,
} => break,
_ => {}
}
}
assert_eq!(
handle.shared_state.catalog_status(),
protocol::WorkerStatus::Idle,
"successful manual compaction must release the execution fence"
);
let command = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot());
let _ = handle.send(Method::Shutdown { command }).await;
} }
+152 -48
View File
@@ -1,5 +1,5 @@
use std::pin::Pin; use std::pin::Pin;
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use agen::Engine; use agen::Engine;
@@ -25,6 +25,15 @@ use worker::{
type TestStore = CombinedStore<FsStore, FsWorkerStore>; type TestStore = CombinedStore<FsStore, FsWorkerStore>;
static NEXT_COMMAND_ID: AtomicU64 = AtomicU64::new(1);
fn worker_command(handle: &WorkerHandle) -> protocol::WorkerCommandEnvelope {
protocol::WorkerCommandEnvelope::for_snapshot(
NEXT_COMMAND_ID.fetch_add(1, Ordering::Relaxed),
&handle.shared_state.snapshot(),
)
}
/// Reconstruct a worker-history-like `Vec<Item>` from the live session /// Reconstruct a worker-history-like `Vec<Item>` from the live session
/// log mirror held by the Worker's broadcast sink. Replaces the previous /// log mirror held by the Worker's broadcast sink. Replaces the previous
/// `WorkerSharedState.history()` test helper now that the mirror lives in /// `WorkerSharedState.history()` test helper now that the mirror lives in
@@ -313,7 +322,12 @@ async fn controller_grants_read_scope_for_exact_bash_output_directory() {
})); }));
assert!(!handle.runtime_dir.path().join("bash-output").exists()); assert!(!handle.runtime_dir.path().join("bash-output").exists());
handle.send(Method::Shutdown).await.unwrap(); handle
.send(Method::Shutdown {
command: worker_command(&handle),
})
.await
.unwrap();
shutdown_rx.await.unwrap(); shutdown_rx.await.unwrap();
} }
@@ -346,7 +360,12 @@ async fn shutdown_closes_bound_workdir_session() {
WorkerController::spawn(worker, runtime_base.path(), &bash_output_dir) WorkerController::spawn(worker, runtime_base.path(), &bash_output_dir)
.await .await
.unwrap(); .unwrap();
handle.send(Method::Shutdown).await.unwrap(); handle
.send(Method::Shutdown {
command: worker_command(&handle),
})
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx) tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
.await .await
.expect("controller should shut down") .expect("controller should shut down")
@@ -461,7 +480,12 @@ async fn controller_projects_workdir_command_events_and_snapshot_state() {
!durable_history.contains("ready") && !durable_history.contains("done"), !durable_history.contains("ready") && !durable_history.contains("done"),
"operational command chunks must not be appended to Worker history: {durable_history}" "operational command chunks must not be appended to Worker history: {durable_history}"
); );
handle.send(Method::Shutdown).await.unwrap(); handle
.send(Method::Shutdown {
command: worker_command(&handle),
})
.await
.unwrap();
} }
#[tokio::test] #[tokio::test]
@@ -533,7 +557,12 @@ async fn controller_refreshes_command_snapshot_after_high_output_provider_lag()
.await .await
.unwrap(); .unwrap();
assert_eq!(output.status, workdir::CommandStatus::Cancelled); assert_eq!(output.status, workdir::CommandStatus::Cancelled);
handle.send(Method::Shutdown).await.unwrap(); handle
.send(Method::Shutdown {
command: worker_command(&handle),
})
.await
.unwrap();
} }
#[tokio::test] #[tokio::test]
@@ -575,13 +604,13 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) { async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop { loop {
if handle.shared_state.get_status() == status { if handle.shared_state.catalog_status() == status {
return; return;
} }
assert!( assert!(
tokio::time::Instant::now() < deadline, tokio::time::Instant::now() < deadline,
"timed out waiting for status {status:?}; current={:?}", "timed out waiting for status {status:?}; current={:?}",
handle.shared_state.get_status() handle.shared_state.catalog_status()
); );
tokio::time::sleep(std::time::Duration::from_millis(10)).await; tokio::time::sleep(std::time::Duration::from_millis(10)).await;
} }
@@ -1033,7 +1062,8 @@ async fn run_end_returns_to_idle_without_busy_status() {
Ok(Event::RunEnd { result: protocol::RunResult::Finished }) => { Ok(Event::RunEnd { result: protocol::RunResult::Finished }) => {
saw_run_end = true; saw_run_end = true;
} }
Ok(Event::Status { status: WorkerStatus::Idle }) if saw_run_end => { Ok(Event::WorkerState { snapshot })
if saw_run_end && snapshot.catalog_status() == WorkerStatus::Idle => {
saw_idle_status = true; saw_idle_status = true;
break; break;
} }
@@ -1050,7 +1080,7 @@ async fn run_end_returns_to_idle_without_busy_status() {
saw_idle_status, saw_idle_status,
"expected idle status immediately after RunEnd" "expected idle status immediately after RunEnd"
); );
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
} }
#[tokio::test] #[tokio::test]
@@ -1128,9 +1158,7 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
loop { loop {
if matches!( if matches!(
events.recv().await, events.recv().await,
Ok(Event::Status { Ok(Event::WorkerState { snapshot }) if snapshot.catalog_status() == WorkerStatus::Running
status: WorkerStatus::Running,
})
) { ) {
break; break;
} }
@@ -1205,8 +1233,8 @@ async fn attach_snapshot_includes_current_status() {
loop { loop {
let event = reader.next::<Event>().await.unwrap().unwrap(); let event = reader.next::<Event>().await.unwrap().unwrap();
match event { match event {
Event::Snapshot { status, .. } => { Event::Snapshot { state, .. } => {
assert_eq!(status, WorkerStatus::Running); assert_eq!(state.catalog_status(), WorkerStatus::Running);
return; return;
} }
Event::Alert(_) => continue, Event::Alert(_) => continue,
@@ -1221,7 +1249,7 @@ async fn shared_state_starts_idle() {
let worker = make_worker(client).await; let worker = make_worker(client).await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
} }
#[tokio::test] #[tokio::test]
@@ -1241,7 +1269,7 @@ async fn run_updates_shared_state_to_idle_after_completion() {
// Wait for the run to complete // Wait for the run to complete
tokio::time::sleep(std::time::Duration::from_millis(100)).await; tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
} }
#[tokio::test] #[tokio::test]
@@ -1364,7 +1392,12 @@ async fn submit_while_running_is_durably_queued() {
assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued)); assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued));
let pending_snapshot = pending_snapshot.expect("pending snapshot"); let pending_snapshot = pending_snapshot.expect("pending snapshot");
assert_eq!(pending_snapshot.submissions.len(), 1); assert_eq!(pending_snapshot.submissions.len(), 1);
handle.send(Method::Pause).await.unwrap(); handle
.send(Method::Pause {
command: worker_command(&handle),
})
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Paused).await; wait_for_status(&handle, WorkerStatus::Paused).await;
handle handle
.send(Method::ContinuePending { .send(Method::ContinuePending {
@@ -1386,17 +1419,22 @@ async fn submit_while_running_is_durably_queued() {
.await .await
.expect("paused ContinuePending rejection"); .expect("paused ContinuePending rejection");
assert!(rejection.contains("Resume or Cancel")); assert!(rejection.contains("Resume or Cancel"));
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
} }
#[tokio::test] #[tokio::test]
async fn resume_without_pause_returns_error() { async fn resume_without_pause_returns_invalid_state_acknowledgement() {
let client = MockClient::new(simple_text_events()); let client = MockClient::new(simple_text_events());
let worker = make_worker(client).await; let worker = make_worker(client).await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::Resume).await.unwrap(); handle
.send(Method::Resume {
command: worker_command(&handle),
})
.await
.unwrap();
let mut saw_not_paused = false; let mut saw_not_paused = false;
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
@@ -1404,7 +1442,10 @@ async fn resume_without_pause_returns_error() {
tokio::select! { tokio::select! {
event = rx.recv() => { event = rx.recv() => {
match event { match event {
Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotPaused => { Ok(Event::CommandAcknowledged { acknowledgement })
if acknowledgement.command == protocol::WorkerCommandKind::Resume
&& acknowledgement.disposition
== protocol::WorkerCommandDisposition::InvalidState => {
saw_not_paused = true; saw_not_paused = true;
break; break;
} }
@@ -1416,17 +1457,22 @@ async fn resume_without_pause_returns_error() {
} }
} }
assert!(saw_not_paused, "should see not_paused error"); assert!(saw_not_paused, "should see invalid-state acknowledgement");
} }
#[tokio::test] #[tokio::test]
async fn cancel_without_run_returns_error() { async fn cancel_without_run_returns_invalid_state_acknowledgement() {
let client = MockClient::new(simple_text_events()); let client = MockClient::new(simple_text_events());
let worker = make_worker(client).await; let worker = make_worker(client).await;
let handle = spawn_controller(worker).await; let handle = spawn_controller(worker).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
handle.send(Method::Cancel).await.unwrap(); handle
.send(Method::Cancel {
command: worker_command(&handle),
})
.await
.unwrap();
let mut saw_not_running = false; let mut saw_not_running = false;
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
@@ -1434,7 +1480,10 @@ async fn cancel_without_run_returns_error() {
tokio::select! { tokio::select! {
event = rx.recv() => { event = rx.recv() => {
match event { match event {
Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotRunning => { Ok(Event::CommandAcknowledged { acknowledgement })
if acknowledgement.command == protocol::WorkerCommandKind::Cancel
&& acknowledgement.disposition
== protocol::WorkerCommandDisposition::InvalidState => {
saw_not_running = true; saw_not_running = true;
break; break;
} }
@@ -1446,7 +1495,7 @@ async fn cancel_without_run_returns_error() {
} }
} }
assert!(saw_not_running, "should see not_running error"); assert!(saw_not_running, "should see invalid-state acknowledgement");
} }
#[tokio::test] #[tokio::test]
@@ -1822,7 +1871,7 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
} }
tokio::time::sleep(std::time::Duration::from_millis(100)).await; tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
assert!( assert!(
client_for_assert.captured_requests().is_empty(), client_for_assert.captured_requests().is_empty(),
"weak Notify must not stage RunForNotification while idle" "weak Notify must not stage RunForNotification while idle"
@@ -1919,7 +1968,7 @@ async fn worker_event_turn_ended_while_idle_auto_starts_turn_and_injects_system_
saw_worker_event_in_mirror, saw_worker_event_in_mirror,
"Method::WorkerEvent should commit a SystemItem::WorkerEvent entry" "Method::WorkerEvent should commit a SystemItem::WorkerEvent entry"
); );
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
let requests = client_for_assert.captured_requests(); let requests = client_for_assert.captured_requests();
assert_eq!( assert_eq!(
@@ -1982,7 +2031,7 @@ async fn worker_event_scope_sub_delegated_while_idle_stays_control_plane_only()
tokio::time::sleep(std::time::Duration::from_millis(100)).await; tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert_eq!( assert_eq!(
handle.shared_state.get_status(), handle.shared_state.catalog_status(),
WorkerStatus::Idle, WorkerStatus::Idle,
"control-plane ScopeSubDelegated must not auto-start the parent LLM" "control-plane ScopeSubDelegated must not auto-start the parent LLM"
); );
@@ -2085,7 +2134,12 @@ async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() {
.await .await
.unwrap(); .unwrap();
} }
handle.send(Method::Cancel).await.unwrap(); handle
.send(Method::Cancel {
command: worker_command(&handle),
})
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let mut rx = handle.subscribe(); let mut rx = handle.subscribe();
@@ -2482,7 +2536,12 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
"text_delta should arrive before pause" "text_delta should arrive before pause"
); );
handle.send(Method::Pause).await.unwrap(); handle
.send(Method::Pause {
command: worker_command(&handle),
})
.await
.unwrap();
// The controller emits RunEnd { Paused } when the // The controller emits RunEnd { Paused } when the
// EngineError::Cancelled is translated under pause_requested. // EngineError::Cancelled is translated under pause_requested.
@@ -2498,9 +2557,14 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
); );
tokio::time::sleep(std::time::Duration::from_millis(50)).await; tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
handle.send(Method::Resume).await.unwrap(); handle
.send(Method::Resume {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
@@ -2514,7 +2578,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
); );
tokio::time::sleep(std::time::Duration::from_millis(50)).await; tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
// History consistency: exactly [user "hello", assistant // History consistency: exactly [user "hello", assistant
// "resumed output"]. No artifacts from the aborted stream // "resumed output"]. No artifacts from the aborted stream
@@ -2614,7 +2678,12 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
"tool_call_done should arrive before pause" "tool_call_done should arrive before pause"
); );
handle.send(Method::Pause).await.unwrap(); handle
.send(Method::Pause {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
e, e,
@@ -2626,7 +2695,7 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
"expected RunEnd::Paused" "expected RunEnd::Paused"
); );
tokio::time::sleep(std::time::Duration::from_millis(50)).await; tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
// New user input while Paused → `Worker::run` observes // New user input while Paused → `Worker::run` observes
// `last_run_interrupted` and runs its interrupt-prep step, which // `last_run_interrupted` and runs its interrupt-prep step, which
@@ -2785,7 +2854,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
"tool_call_done should arrive before pause" "tool_call_done should arrive before pause"
); );
handle.send(Method::Pause).await.unwrap(); handle
.send(Method::Pause {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
e, e,
@@ -2798,7 +2872,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
); );
wait_for_status(&handle, WorkerStatus::Paused).await; wait_for_status(&handle, WorkerStatus::Paused).await;
handle.send(Method::Cancel).await.unwrap(); handle
.send(Method::Cancel {
command: worker_command(&handle),
})
.await
.unwrap();
wait_for_status(&handle, WorkerStatus::Idle).await; wait_for_status(&handle, WorkerStatus::Idle).await;
let (entries_after_cancel, _rx_after_cancel) = handle.sink.subscribe_with_snapshot(); let (entries_after_cancel, _rx_after_cancel) = handle.sink.subscribe_with_snapshot();
assert!( assert!(
@@ -2824,17 +2903,22 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
"paused cancel must not resume or start another LLM request" "paused cancel must not resume or start another LLM request"
); );
handle.send(Method::Resume).await.unwrap(); handle
.send(Method::Resume {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
e, e,
Event::Error { Event::CommandAcknowledged { acknowledgement }
code: worker::ErrorCode::NotPaused, if acknowledgement.command == protocol::WorkerCommandKind::Resume
.. && acknowledgement.disposition
} == protocol::WorkerCommandDisposition::InvalidState
)) ))
.await, .await,
"resume after paused cancel should be rejected as not paused" "resume after paused cancel should receive invalid-state acknowledgement"
); );
assert_eq!( assert_eq!(
client_for_assert.captured_requests().len(), client_for_assert.captured_requests().len(),
@@ -2943,7 +3027,12 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() {
.await .await
.unwrap(); .unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
handle.send(Method::Cancel).await.unwrap(); handle
.send(Method::Cancel {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
@@ -2981,7 +3070,12 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() {
.await .await
.unwrap(); .unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
handle.send(Method::Pause).await.unwrap(); handle
.send(Method::Pause {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
@@ -3038,7 +3132,12 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() {
.await .await
.unwrap(); .unwrap();
wait_for_status(&handle, WorkerStatus::Running).await; wait_for_status(&handle, WorkerStatus::Running).await;
handle.send(Method::Cancel).await.unwrap(); handle
.send(Method::Cancel {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
e, e,
@@ -3095,7 +3194,12 @@ async fn pause_after_assistant_token_does_not_rollback() {
.await, .await,
"assistant token should be visible before pause" "assistant token should be visible before pause"
); );
handle.send(Method::Pause).await.unwrap(); handle
.send(Method::Pause {
command: worker_command(&handle),
})
.await
.unwrap();
assert!( assert!(
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
+5
View File
@@ -1904,7 +1904,12 @@ pub struct WorkerSummary {
#[serde(default)] #[serde(default)]
pub tags: Vec<String>, pub tags: Vec<String>,
pub workspace: WorkerWorkspaceSummary, pub workspace: WorkerWorkspaceSummary,
/// Runtime catalog lifecycle compatibility state. Live foreground state, when
/// available, is carried separately in `worker_state`.
pub state: String, pub state: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "typescript", ts(optional))]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
pub last_seen_at: Option<String>, pub last_seen_at: Option<String>,
#[serde(default)] #[serde(default)]
pub pinned: bool, pub pinned: bool,
+14 -9
View File
@@ -36,8 +36,6 @@ use worker_runtime::config_bundle::{
ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
}; };
use worker_runtime::error::RuntimeError as EmbeddedRuntimeError; use worker_runtime::error::RuntimeError as EmbeddedRuntimeError;
#[cfg(test)]
use worker_runtime::execution::WorkerExecutionRunState;
use worker_runtime::fs_store::FsRuntimeStoreOptions; use worker_runtime::fs_store::FsRuntimeStoreOptions;
use worker_runtime::http_server::{ use worker_runtime::http_server::{
RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER, RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER,
@@ -245,7 +243,10 @@ pub struct WorkerSummary {
#[serde(default)] #[serde(default)]
pub tags: Vec<String>, pub tags: Vec<String>,
pub workspace: WorkerWorkspaceSummary, pub workspace: WorkerWorkspaceSummary,
/// Runtime catalog lifecycle compatibility state.
pub state: String, pub state: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
pub last_seen_at: Option<String>, pub last_seen_at: Option<String>,
#[serde(default)] #[serde(default)]
pub pinned: bool, pub pinned: bool,
@@ -337,6 +338,7 @@ pub(crate) fn workspace_worker_summary(
workspace_id: summary.workspace.workspace_id, workspace_id: summary.workspace.workspace_id,
}, },
state: summary.state, state: summary.state,
worker_state: summary.worker_state,
last_seen_at: summary.last_seen_at, last_seen_at: summary.last_seen_at,
pinned: summary.pinned, pinned: summary.pinned,
retention_state: summary.retention_state, retention_state: summary.retention_state,
@@ -2000,6 +2002,7 @@ impl EmbeddedWorkerRuntime {
workspace_id: summary.workspace_id.clone(), workspace_id: summary.workspace_id.clone(),
}, },
state: embedded_worker_status_label(summary.status).to_string(), state: embedded_worker_status_label(summary.status).to_string(),
worker_state: summary.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -2039,6 +2042,7 @@ impl EmbeddedWorkerRuntime {
workspace_id: detail.workspace_id.clone(), workspace_id: detail.workspace_id.clone(),
}, },
state: embedded_worker_status_label(detail.status).to_string(), state: embedded_worker_status_label(detail.status).to_string(),
worker_state: detail.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -3342,6 +3346,7 @@ impl RemoteWorkerRuntime {
workspace_id: summary.workspace_id.clone(), workspace_id: summary.workspace_id.clone(),
}, },
state: embedded_worker_status_label(summary.status).to_string(), state: embedded_worker_status_label(summary.status).to_string(),
worker_state: summary.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -3385,6 +3390,7 @@ impl RemoteWorkerRuntime {
workspace_id: detail.workspace_id.clone(), workspace_id: detail.workspace_id.clone(),
}, },
state: embedded_worker_status_label(detail.status).to_string(), state: embedded_worker_status_label(detail.status).to_string(),
worker_state: detail.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -4732,6 +4738,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> WorkerSummary {
workspace_id: None, workspace_id: None,
}, },
state: "unsupported".to_string(), state: "unsupported".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -5170,7 +5177,6 @@ mod tests {
request.worker_ref, request.worker_ref,
self.backend_id(), self.backend_id(),
), ),
run_state: WorkerExecutionRunState::Idle,
working_directory: request working_directory: request
.working_directory .working_directory
.as_ref() .as_ref()
@@ -5199,8 +5205,8 @@ mod tests {
let content = input.content; let content = input.content;
std::thread::spawn(move || { std::thread::spawn(move || {
std::thread::sleep(std::time::Duration::from_millis(10)); std::thread::sleep(std::time::Duration::from_millis(10));
let _ = context.publish_protocol_event(protocol::Event::Status { let _ = context.publish_protocol_event(protocol::Event::WorkerState {
status: protocol::WorkerStatus::Running, snapshot: protocol::WorkerStatus::Running.into(),
}); });
let _ = context.publish_protocol_event(protocol::Event::TextDone { let _ = context.publish_protocol_event(protocol::Event::TextDone {
text: format!("echo: {content}"), text: format!("echo: {content}"),
@@ -5208,14 +5214,13 @@ mod tests {
let _ = context.publish_protocol_event(protocol::Event::RunEnd { let _ = context.publish_protocol_event(protocol::Event::RunEnd {
result: protocol::RunResult::Finished, result: protocol::RunResult::Finished,
}); });
let _ = context.publish_protocol_event(protocol::Event::Status { let _ = context.publish_protocol_event(protocol::Event::WorkerState {
status: protocol::WorkerStatus::Idle, snapshot: protocol::WorkerStatus::Idle.into(),
}); });
}); });
if let Some(submission_request_id) = submission_request_id { if let Some(submission_request_id) = submission_request_id {
worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionResult::accepted_submission(
worker_runtime::execution::WorkerExecutionOperation::Input, worker_runtime::execution::WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
submission_request_id, submission_request_id,
uuid::Uuid::now_v7().to_string(), uuid::Uuid::now_v7().to_string(),
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
@@ -5223,7 +5228,6 @@ mod tests {
} else { } else {
worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionResult::accepted(
worker_runtime::execution::WorkerExecutionOperation::Input, worker_runtime::execution::WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
) )
} }
} }
@@ -5256,6 +5260,7 @@ mod tests {
workspace_id: None, workspace_id: None,
}, },
state: "available".to_string(), state: "available".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -6,7 +6,7 @@ use worker_runtime::catalog::{
}; };
use worker_runtime::execution::{ use worker_runtime::execution::{
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult,
WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
}; };
use worker_runtime::identity::WorkerId; use worker_runtime::identity::WorkerId;
use worker_runtime::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary}; use worker_runtime::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary};
@@ -22,7 +22,6 @@ impl WorkerExecutionBackend for TestExecutionBackend {
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::connected( WorkerExecutionSpawnResult::connected(
WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
WorkerExecutionRunState::Idle,
None, None,
) )
} }
@@ -35,24 +34,17 @@ impl WorkerExecutionBackend for TestExecutionBackend {
if let Some(submission_request_id) = input.submission_request_id { if let Some(submission_request_id) = input.submission_request_id {
WorkerExecutionResult::accepted_submission( WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
submission_request_id, submission_request_id,
uuid::Uuid::now_v7().to_string(), uuid::Uuid::now_v7().to_string(),
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
) )
} else { } else {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
WorkerExecutionOperation::Input,
WorkerExecutionRunState::Busy,
)
} }
} }
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
WorkerExecutionResult::accepted( WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
WorkerExecutionOperation::Stop,
WorkerExecutionRunState::Stopped,
)
} }
} }
@@ -199,8 +191,8 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
runtime runtime
.observe_worker_event( .observe_worker_event(
&worker.worker_ref, &worker.worker_ref,
protocol::Event::Status { protocol::Event::WorkerState {
status: protocol::WorkerStatus::Running, snapshot: protocol::WorkerStatus::Running.into(),
}, },
) )
.unwrap(); .unwrap();
@@ -210,7 +202,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
BrokerSubscriptionEvent::Event { BrokerSubscriptionEvent::Event {
payload: SubscriptionEventPayload::WorkerUpserted { ref worker }, payload: SubscriptionEventPayload::WorkerUpserted { ref worker },
.. ..
} if worker.state == SubscriptionWorkerState::Running } if worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running
)),
..
})
)
)); ));
} }
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap(); let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
@@ -220,7 +221,18 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
assert!(matches!( assert!(matches!(
snapshot, snapshot,
SubscriptionSnapshot::Workers { workers } SubscriptionSnapshot::Workers { workers }
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) if workers.iter().any(|worker| {
worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
),
..
})
)
})
)); ));
drop(late); drop(late);
@@ -339,14 +351,24 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
runtime runtime
.observe_worker_event( .observe_worker_event(
&worker.worker_ref, &worker.worker_ref,
protocol::Event::Status { protocol::Event::WorkerState {
status: protocol::WorkerStatus::Running, snapshot: protocol::WorkerStatus::Running.into(),
}, },
) )
.unwrap(); .unwrap();
assert!(matches!(next_event(&mut subscription).await, assert!(matches!(next_event(&mut subscription).await,
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. } BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running)); if worker.runtime_id.as_deref() == Some("embedded-worker-runtime")
&& worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running
)),
..
})
)));
let mut late = broker let mut late = broker
.subscribe( .subscribe(
"embedded-worker-runtime", "embedded-worker-runtime",
@@ -359,7 +381,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
assert!(matches!( assert!(matches!(
snapshot, snapshot,
SubscriptionSnapshot::Workers { workers } SubscriptionSnapshot::Workers { workers }
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) if workers.iter().any(|worker| {
worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
),
..
})
)
})
)); ));
runtime runtime
+42 -9
View File
@@ -15766,6 +15766,7 @@ fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> WorkerSummary
singleton_key: None, singleton_key: None,
tags: Vec::new(), tags: Vec::new(),
state: "missing".to_string(), state: "missing".to_string(),
worker_state: None,
last_seen_at: Some(record.updated_at.clone()), last_seen_at: Some(record.updated_at.clone()),
pinned: record.retention_state == "pinned", pinned: record.retention_state == "pinned",
retention_state: record.retention_state.clone(), retention_state: record.retention_state.clone(),
@@ -19422,7 +19423,6 @@ mod tests {
request.worker_ref, request.worker_ref,
self.backend_id(), self.backend_id(),
), ),
run_state: worker_runtime::execution::WorkerExecutionRunState::Idle,
working_directory, working_directory,
} }
} }
@@ -19438,17 +19438,43 @@ mod tests {
.push((handle.worker_ref().clone(), method)); .push((handle.worker_ref().clone(), method));
worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionResult::accepted(
worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod, worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod,
worker_runtime::execution::WorkerExecutionRunState::Idle,
) )
} }
fn worker_snapshot(
&self,
handle: &worker_runtime::execution::WorkerExecutionHandle,
) -> Option<protocol::Event> {
Some(protocol::Event::Snapshot {
session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: Vec::new(),
},
greeting: protocol::Greeting {
worker_name: handle.worker_ref().worker_id.to_string(),
cwd: String::new(),
provider: "deterministic-workspace-server-test".to_string(),
model: "deterministic-workspace-server-test".to_string(),
scope_summary: "test execution snapshot".to_string(),
tools: Vec::new(),
context_window: 0,
context_tokens: 0,
},
state: protocol::WorkerStateSnapshot::initial(1),
in_flight: protocol::InFlightSnapshot {
blocks: Vec::new(),
commands: Vec::new(),
},
internal_workers: Vec::new(),
})
}
fn stop_worker( fn stop_worker(
&self, &self,
_handle: &worker_runtime::execution::WorkerExecutionHandle, _handle: &worker_runtime::execution::WorkerExecutionHandle,
) -> worker_runtime::execution::WorkerExecutionResult { ) -> worker_runtime::execution::WorkerExecutionResult {
worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionResult::accepted(
worker_runtime::execution::WorkerExecutionOperation::Stop, worker_runtime::execution::WorkerExecutionOperation::Stop,
worker_runtime::execution::WorkerExecutionRunState::Stopped,
) )
} }
@@ -19458,7 +19484,6 @@ mod tests {
) -> worker_runtime::execution::WorkerExecutionResult { ) -> worker_runtime::execution::WorkerExecutionResult {
worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionResult::accepted(
worker_runtime::execution::WorkerExecutionOperation::Cancel, worker_runtime::execution::WorkerExecutionOperation::Cancel,
worker_runtime::execution::WorkerExecutionRunState::Stopped,
) )
} }
@@ -19495,16 +19520,16 @@ mod tests {
if let Some(submission_request_id) = submission_request_id { if let Some(submission_request_id) = submission_request_id {
worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionResult::accepted_submission(
worker_runtime::execution::WorkerExecutionOperation::Input, worker_runtime::execution::WorkerExecutionOperation::Input,
worker_runtime::execution::WorkerExecutionRunState::Idle,
submission_request_id, submission_request_id,
uuid::Uuid::now_v7().to_string(), uuid::Uuid::now_v7().to_string(),
protocol::SubmissionDisposition::Started, protocol::SubmissionDisposition::Started,
) )
.with_worker_state(protocol::WorkerStateSnapshot::initial(1))
} else { } else {
worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionResult::accepted(
worker_runtime::execution::WorkerExecutionOperation::Input, worker_runtime::execution::WorkerExecutionOperation::Input,
worker_runtime::execution::WorkerExecutionRunState::Idle,
) )
.with_worker_state(protocol::WorkerStateSnapshot::initial(1))
} }
} }
} }
@@ -25570,6 +25595,7 @@ mod tests {
workspace_id: Some(TEST_WORKSPACE_ID.to_string()), workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
}, },
state: "idle".to_string(), state: "idle".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "normal".to_string(), retention_state: "normal".to_string(),
@@ -25666,6 +25692,7 @@ mod tests {
workspace_id: Some(TEST_WORKSPACE_ID.to_string()), workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
}, },
state: "idle".to_string(), state: "idle".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "normal".to_string(), retention_state: "normal".to_string(),
@@ -28556,7 +28583,13 @@ mod tests {
protocol::subscription::SubscriptionFramePayload::WorkerProtocol( protocol::subscription::SubscriptionFramePayload::WorkerProtocol(
protocol::subscription::SubscriptionWorkerProtocolMethod { protocol::subscription::SubscriptionWorkerProtocolMethod {
subscription_id: second_protocol_subscription_id, subscription_id: second_protocol_subscription_id,
method: protocol::Method::Resume, method: protocol::Method::Resume {
command: protocol::WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 1,
expected_worker_state_revision: 0,
},
},
}, },
), ),
); );
@@ -28573,7 +28606,7 @@ mod tests {
.iter() .iter()
.any(|(worker_ref, method)| { .any(|(worker_ref, method)| {
worker_ref.worker_id.to_string() == worker_id worker_ref.worker_id.to_string() == worker_id
&& matches!(method, protocol::Method::Resume) && matches!(method, protocol::Method::Resume { .. })
}) })
{ {
break; break;
@@ -28586,7 +28619,7 @@ mod tests {
let protocol_methods = execution_backend.protocol_methods(); let protocol_methods = execution_backend.protocol_methods();
assert!(protocol_methods.iter().any(|(worker_ref, method)| { assert!(protocol_methods.iter().any(|(worker_ref, method)| {
worker_ref.worker_id.to_string() == worker_id worker_ref.worker_id.to_string() == worker_id
&& matches!(method, protocol::Method::Resume) && matches!(method, protocol::Method::Resume { .. })
})); }));
server.abort(); server.abort();
} }
+49 -4
View File
@@ -10,6 +10,37 @@ export type CompletionKind = "file";
export type WorkerStatus = "idle" | "running" | "paused" | "stopped"; export type WorkerStatus = "idle" | "running" | "paused" | "stopped";
export type WorkerCommandEnvelope = {
/**
* Caller-owned sequence. A controller accepts command ids in strictly
* increasing order for one execution generation.
*/
command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, };
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
/**
* The complete authoritative state observed after command admission.
*/
state: WorkerStateSnapshot, };
export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
export type WorkerMaintenanceState = "compacting";
export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "kind": "maintenance", "state": WorkerMaintenanceState };
export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState };
export type WorkerStateSnapshot = { execution_generation: number, revision: number,
/**
* Highest lifecycle command id observed by this controller generation.
*/
last_command_id: number, state: WorkerState, };
export type TurnResult = "finished" | "paused"; export type TurnResult = "finished" | "paused";
export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup"; export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup";
@@ -202,7 +233,16 @@ resource_key?: string | null,
/** /**
* Producer-owned monotonic revision for this Worker subject. * Producer-owned monotonic revision for this Worker subject.
*/ */
subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null, subject_revision: number,
/**
* Latest revisioned foreground state observed from the Worker. This remains
* absent until an authoritative Worker snapshot/event has been applied.
*/
worker_state?: WorkerStateSnapshot | null,
/**
* Runtime catalog lifecycle compatibility projection; not foreground-state authority.
*/
state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
/** /**
* Workspace-facing Repository key. Runtime producers leave this unset and * Workspace-facing Repository key. Runtime producers leave this unset and
* Workspace Server projections replace `repository_id` with this field. * Workspace Server projections replace `repository_id` with this field.
@@ -231,7 +271,7 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip
export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }); export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod });
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
/** /**
@@ -247,7 +287,12 @@ summary: string,
* Full tool output. Absent when the tool chose to return * Full tool output. Absent when the tool chose to return
* summary-only, or when the result was pruned. * summary-only, or when the result was pruned.
*/ */
output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, status: WorkerStatus, output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting,
/**
* Full revisioned live execution state. `Stopped` remains Runtime
* catalog authority and is deliberately not represented here.
*/
state: WorkerStateSnapshot,
/** /**
* Unfinished model output that has already streamed in the current * Unfinished model output that has already streamed in the current
* run but is not yet represented by committed snapshot entries. * run but is not yet represented by committed snapshot entries.
@@ -257,4 +302,4 @@ in_flight?: InFlightSnapshot,
* Parent-owned Internal Worker sessions visible to this client. * Parent-owned Internal Worker sessions visible to this client.
* Service-private Internal Workers are deliberately excluded. * Service-private Internal Workers are deliberately excluded.
*/ */
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" }; internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" };
@@ -1,4 +1,4 @@
import type { Event } from "$lib/generated/protocol"; import type { Event, WorkerStateSnapshot, WorkerStatus } from "$lib/generated/protocol";
import { import {
type ConsoleEventInput, type ConsoleEventInput,
type ConsoleLine, type ConsoleLine,
@@ -19,6 +19,23 @@ declare const Deno: {
test(name: string, fn: () => void): void; test(name: string, fn: () => void): void;
}; };
function workerState(status: WorkerStatus): WorkerStateSnapshot {
return {
execution_generation: 1,
revision: status === "idle" ? 0 : 1,
last_command_id: 0,
state: status === "idle"
? { kind: "idle" }
: {
kind: "busy",
state: {
kind: "run",
state: status === "paused" ? "paused" : "running",
},
},
};
}
function assert(condition: unknown, message: string): asserts condition { function assert(condition: unknown, message: string): asserts condition {
if (!condition) { if (!condition) {
throw new Error(message); throw new Error(message);
@@ -131,7 +148,7 @@ function snapshotEvent(cwd: string, entries: unknown[] = []): Event {
context_window: 100, context_window: 100,
context_tokens: 20, context_tokens: 20,
}, },
status: "idle", state: workerState("idle"),
in_flight: { blocks: [] }, in_flight: { blocks: [] },
}, },
}; };
@@ -201,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", ()
); );
}); });
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
const projector = createConsoleProjector();
const running: WorkerStateSnapshot = {
execution_generation: 4,
revision: 3,
last_command_id: 2,
state: { kind: "busy", state: { kind: "run", state: "running" } },
};
const paused: WorkerStateSnapshot = {
...running,
revision: 4,
last_command_id: 3,
state: { kind: "busy", state: { kind: "run", state: "paused" } },
};
let projection = projector.append([
{
eventId: "running",
event: { event: "worker_state", data: { snapshot: running } },
},
{
eventId: "stale",
event: {
event: "worker_state",
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
},
},
{
eventId: "pause-ack",
event: {
event: "command_acknowledged",
data: {
acknowledgement: {
command_id: 3,
command: "pause",
disposition: "accepted",
state: paused,
},
},
},
},
]);
assertEquals(projection.workerState, paused);
assertEquals(projection.status, "paused");
projection = projector.append([{
eventId: "conflict",
event: {
event: "worker_state",
data: { snapshot: { ...paused, state: { kind: "idle" } } },
},
}]);
assertEquals(projection.workerState, paused);
assert(
projection.lines.some((line) =>
line.eventId === "conflict:worker-state-conflict" && line.error
),
"conflicting equal-version snapshots must fail closed",
);
});
Deno.test("snapshot replaces a live error with one durable run_errored row", () => { Deno.test("snapshot replaces a live error with one durable run_errored row", () => {
const projector = createConsoleProjector(); const projector = createConsoleProjector();
let projection = projector.append([ let projection = projector.append([
@@ -213,7 +290,7 @@ Deno.test("snapshot replaces a live error with one durable run_errored row", ()
}, },
{ {
eventId: "idle-after-error", eventId: "idle-after-error",
event: { event: "status", data: { status: "idle" } } satisfies Event, event: { event: "worker_state", data: { snapshot: workerState("idle") } } satisfies Event,
}, },
]); ]);
@@ -653,7 +730,7 @@ Deno.test("projectConsole streams distinct Bash stdout and stderr through termin
Deno.test("snapshot restores bounded in-flight Bash command output", () => { Deno.test("snapshot restores bounded in-flight Bash command output", () => {
const snapshot = snapshotEvent("/repo"); const snapshot = snapshotEvent("/repo");
if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected"); if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected");
snapshot.data.status = "running"; snapshot.data.state = workerState("running");
snapshot.data.in_flight = { snapshot.data.in_flight = {
blocks: [{ blocks: [{
kind: "tool_call", kind: "tool_call",
@@ -1403,7 +1480,7 @@ Deno.test("projectConsole hides lifecycle events and renders system items", () =
const projection = projectConsole([ const projection = projectConsole([
{ {
eventId: "30", eventId: "30",
event: { event: "status", data: { status: "running" } } satisfies Event, event: { event: "worker_state", data: { snapshot: workerState("running") } } satisfies Event,
}, },
{ {
eventId: "31", eventId: "31",
@@ -1527,7 +1604,7 @@ Deno.test("projectConsole renders snapshot entries and in-flight output", () =>
context_window: 100, context_window: 100,
context_tokens: 20, context_tokens: 20,
}, },
status: "running", state: workerState("running"),
in_flight: { in_flight: {
blocks: [ blocks: [
{ kind: "text", text: "partial" }, { kind: "text", text: "partial" },
@@ -1578,7 +1655,7 @@ Deno.test("projectConsole restores system items from snapshot entries", () => {
context_window: 100, context_window: 100,
context_tokens: 20, context_tokens: 20,
}, },
status: "idle", state: workerState("idle"),
}, },
} satisfies Event, } satisfies Event,
}]); }]);
@@ -1922,7 +1999,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
kind: "sub_worker", kind: "sub_worker",
}, },
revision: 1, revision: 1,
event: { event: "status", data: { status: "running" } }, event: { event: "worker_state", data: { snapshot: workerState("running") } },
}, },
}, },
}, },
@@ -1941,7 +2018,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
kind: "sub_worker", kind: "sub_worker",
}, },
revision: 1, revision: 1,
event: { event: "status", data: { status: "idle" } }, event: { event: "worker_state", data: { snapshot: workerState("idle") } },
}, },
}, },
}]); }]);
@@ -2033,7 +2110,7 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections"
kind: "sub_worker", kind: "sub_worker",
}, },
revision: 1, revision: 1,
event: { event: "status", data: { status: "running" } }, event: { event: "worker_state", data: { snapshot: workerState("running") } },
}, },
}, },
}]); }]);
@@ -10,6 +10,9 @@ import type {
InternalWorkerRef, InternalWorkerRef,
InternalWorkerSnapshot, InternalWorkerSnapshot,
Segment, Segment,
WorkerState,
WorkerStateSnapshot,
WorkerStatus,
} from "$lib/generated/protocol"; } from "$lib/generated/protocol";
import { stringify as stringifyYaml } from "yaml"; import { stringify as stringifyYaml } from "yaml";
import { workspaceRoute } from "$lib/workspace/api/http"; import { workspaceRoute } from "$lib/workspace/api/http";
@@ -169,6 +172,7 @@ export type ConsoleProjection = {
tasks: ConsoleTask[]; tasks: ConsoleTask[];
taskNextId: number; taskNextId: number;
status: string | null; status: string | null;
workerState: WorkerStateSnapshot | null;
usage: string | null; usage: string | null;
runActivity: RunActivityStats; runActivity: RunActivityStats;
cwd: string | null; cwd: string | null;
@@ -251,12 +255,22 @@ export function isConsoleProjectionEvent(event: ProtocolEvent): boolean {
return event.event !== "completions"; return event.event !== "completions";
} }
function workerStatusFromState(snapshot: WorkerStateSnapshot): WorkerStatus {
if (snapshot.state.kind === "idle") return "idle";
if (
snapshot.state.state.kind === "run" &&
snapshot.state.state.state === "paused"
) return "paused";
return "running";
}
export function emptyConsoleProjection(): ConsoleProjection { export function emptyConsoleProjection(): ConsoleProjection {
return { return {
lines: [], lines: [],
tasks: [], tasks: [],
taskNextId: 1, taskNextId: 1,
status: null, status: null,
workerState: null,
usage: null, usage: null,
runActivity: emptyRunActivityStats(), runActivity: emptyRunActivityStats(),
cwd: null, cwd: null,
@@ -783,6 +797,60 @@ function refreshCompactionActivity(
return changed ? { ...projection, lines } : projection; return changed ? { ...projection, lines } : projection;
} }
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
if (left.kind !== right.kind) return false;
if (left.kind === "idle" || right.kind === "idle") return true;
return left.state.kind === right.state.kind &&
left.state.state === right.state.state;
}
function workerStateSnapshotEqual(
left: WorkerStateSnapshot,
right: WorkerStateSnapshot,
): boolean {
return left.execution_generation === right.execution_generation &&
left.revision === right.revision &&
left.last_command_id === right.last_command_id &&
workerStateEqual(left.state, right.state);
}
function applyWorkerStateSnapshot(
projection: ConsoleProjection,
incoming: WorkerStateSnapshot,
eventId: string,
): void {
const current = projection.workerState;
if (!current) {
projection.workerState = incoming;
projection.status = workerStatusFromState(incoming);
return;
}
const generationOrder = incoming.execution_generation -
current.execution_generation;
const revisionOrder = incoming.revision - current.revision;
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
projection.workerState = incoming;
projection.status = workerStatusFromState(incoming);
return;
}
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
return;
}
if (!workerStateSnapshotEqual(current, incoming)) {
projection.lines.push(
line(
`${eventId}:worker-state-conflict`,
"error",
"error · internal",
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
undefined,
false,
true,
),
);
}
}
export function applyProtocolEvent( export function applyProtocolEvent(
projection: ConsoleProjection, projection: ConsoleProjection,
envelope: ConsoleEventInput, envelope: ConsoleEventInput,
@@ -793,6 +861,7 @@ export function applyProtocolEvent(
tasks: [...projection.tasks], tasks: [...projection.tasks],
taskNextId: projection.taskNextId, taskNextId: projection.taskNextId,
status: projection.status, status: projection.status,
workerState: projection.workerState,
usage: projection.usage, usage: projection.usage,
runActivity: applyRunActivityEvent( runActivity: applyRunActivityEvent(
projection.runActivity, projection.runActivity,
@@ -903,7 +972,6 @@ export function applyProtocolEvent(
); );
break; break;
case "snapshot": { case "snapshot": {
next.status = event.data.status;
next.cwd = event.data.greeting.cwd; next.cwd = event.data.greeting.cwd;
const snapshot = snapshotProjectionFromSession( const snapshot = snapshotProjectionFromSession(
envelope.eventId, envelope.eventId,
@@ -953,6 +1021,7 @@ export function applyProtocolEvent(
}; };
} }
} }
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
break; break;
} }
case "internal_worker": { case "internal_worker": {
@@ -1000,8 +1069,15 @@ export function applyProtocolEvent(
if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1); if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1);
break; break;
} }
case "status": case "worker_state":
next.status = event.data.status; applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
break;
case "command_acknowledged":
applyWorkerStateSnapshot(
next,
event.data.acknowledgement.state,
envelope.eventId,
);
break; break;
case "command": case "command":
applyCommandEvent(next, envelope.eventId, event.data.event); applyCommandEvent(next, envelope.eventId, event.data.event);
@@ -1939,6 +2015,7 @@ function snapshotProjectionFromSession(
tasks: [], tasks: [],
taskNextId: 1, taskNextId: 1,
status: null, status: null,
workerState: null,
usage: null, usage: null,
runActivity: emptyRunActivityStats(), runActivity: emptyRunActivityStats(),
cwd, cwd,
@@ -75,7 +75,12 @@ Deno.test("new invoke and running snapshot reset run activity", () => {
data: { data: {
entries: [], entries: [],
greeting: { text: "", profile: "" }, greeting: { text: "", profile: "" },
status: "idle", state: {
execution_generation: 1,
revision: 0,
last_command_id: 0,
state: { kind: "idle" },
},
in_flight: {}, in_flight: {},
internal_workers: [], internal_workers: [],
}, },
@@ -25,7 +25,9 @@ export function applyRunActivityEvent(
case "invoke_start": case "invoke_start":
return { ...emptyRunActivityStats(), startedAtMs: observedAtMs }; return { ...emptyRunActivityStats(), startedAtMs: observedAtMs };
case "snapshot": case "snapshot":
return event.data.status === "running" return event.data.state.state.kind === "busy" &&
!(event.data.state.state.state.kind === "run" &&
event.data.state.state.state.state === "paused")
? { ...emptyRunActivityStats(), startedAtMs: observedAtMs } ? { ...emptyRunActivityStats(), startedAtMs: observedAtMs }
: emptyRunActivityStats(); : emptyRunActivityStats();
case "turn_start": case "turn_start":
@@ -620,7 +620,7 @@ Deno.test("Worker Console paste chips preserve typed draft and target authority"
consolePage.includes("preserveExactText: value.textPastes.length > 0") && consolePage.includes("preserveExactText: value.textPastes.length > 0") &&
consolePage.includes("composerDrafts.set(activeComposerTargetKey") && consolePage.includes("composerDrafts.set(activeComposerTargetKey") &&
consolePage.includes("switchComposerTarget(target)") && consolePage.includes("switchComposerTarget(target)") &&
consolePage.includes('sendControl({ method: "cancel" }, "Stop")'), consolePage.includes('sendWorkerControl("cancel")'),
"Paste chips should use shared threshold classification, atomic keyboard behavior, accessible labels, typed restore, and per-Worker draft authority", "Paste chips should use shared threshold classification, atomic keyboard behavior, accessible labels, typed restore, and per-Worker draft authority",
); );
}); });
@@ -787,7 +787,10 @@ Deno.test("Worker Console page is routed by runtime_id and worker_id through bac
consolePage.includes( consolePage.includes(
'const composerEditable = $derived(protocolState === "open" && !sending);', 'const composerEditable = $derived(protocolState === "open" && !sending);',
) && ) &&
consolePage.includes('sendControl({ method: "cancel" }, "Stop")') && consolePage.includes('sendWorkerControl("cancel")') &&
consolePage.includes("lifecycleMethod(command)") &&
consolePage.includes("expected_worker_state_revision") &&
consolePage.includes("expected_execution_generation") &&
consolePage.includes("onsubmit={handleComposerSubmit}") && consolePage.includes("onsubmit={handleComposerSubmit}") &&
consolePage.includes("disabled={!composerEditable}") && consolePage.includes("disabled={!composerEditable}") &&
consolePage.includes("class:stop={workerRunning}") && consolePage.includes("class:stop={workerRunning}") &&
@@ -19,6 +19,7 @@ import type {
Event as PodProtocolEvent, Event as PodProtocolEvent,
Method as PodProtocolMethod, Method as PodProtocolMethod,
Segment as PodProtocolSegment, Segment as PodProtocolSegment,
WorkerStateSnapshot,
} from "$lib/generated/protocol"; } from "$lib/generated/protocol";
import type { import type {
GitCommitSummary as SharedGitCommitSummary, GitCommitSummary as SharedGitCommitSummary,
@@ -99,6 +100,7 @@ export type Worker = {
tags: string[]; tags: string[];
workspace: { visibility: string; identity: string }; workspace: { visibility: string; identity: string };
state: string; state: string;
worker_state?: WorkerStateSnapshot | null;
pinned?: boolean; pinned?: boolean;
retention_state?: string; retention_state?: string;
last_seen_at?: string | null; last_seen_at?: string | null;
@@ -0,0 +1,12 @@
import type { WorkerStateSnapshot } from "$lib/generated/protocol";
export function liveWorkerState(worker: {
state: string;
worker_state?: WorkerStateSnapshot | null;
}): string {
const state = worker.worker_state?.state;
if (!state) return worker.state === "stopped" ? "stopped" : "unknown";
if (state.kind === "idle") return "idle";
if (state.state.kind === "maintenance") return "running";
return state.state.state === "paused" ? "paused" : "running";
}
@@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void {
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
} }
} }
import { liveWorkerState } from './worker-state';
import { import {
applyWorkspaceWorkersFrame, applyWorkspaceWorkersFrame,
createWorkspaceWorkersProjection, createWorkspaceWorkersProjection,
@@ -33,6 +34,22 @@ function worker(
}; };
} }
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
const active = worker('runtime-a', 'worker-1', 1);
active.worker_state = {
execution_generation: 4,
revision: 2,
last_command_id: 1,
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
};
assertEquals(liveWorkerState(active), 'paused');
const unavailable = worker('runtime-a', 'worker-2', 1);
assertEquals(liveWorkerState(unavailable), 'unknown');
unavailable.state = 'stopped';
assertEquals(liveWorkerState(unavailable), 'stopped');
});
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => { Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
const projection = createWorkspaceWorkersProjection(); const projection = createWorkspaceWorkersProjection();
const frame: SubscriptionFrame = { const frame: SubscriptionFrame = {
@@ -5,6 +5,7 @@ import {
applyWorkspaceWorkersFrame, applyWorkspaceWorkersFrame,
createWorkspaceWorkersProjection, createWorkspaceWorkersProjection,
} from './worker-subscription-model'; } from './worker-subscription-model';
import { liveWorkerState } from './worker-state';
import { compareWorkersForSidebar } from './workers'; import { compareWorkersForSidebar } from './workers';
import type { Worker } from './types'; import type { Worker } from './types';
@@ -90,7 +91,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker {
profile: worker.profile ?? null, profile: worker.profile ?? null,
tags: [], tags: [],
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' }, workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
state: worker.state, state: liveWorkerState(worker),
worker_state: worker.worker_state,
pinned: false, pinned: false,
retention_state: 'transient', retention_state: 'transient',
implementation: { implementation: {
@@ -52,11 +52,7 @@
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store"; import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
import { workspaceApiPath } from "$lib/workspace/api/http"; import { workspaceApiPath } from "$lib/workspace/api/http";
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer"; import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
import type { import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types";
Diagnostic,
Worker,
PodProtocolEvent,
} from "$lib/workspace/sidebar/types";
type Props = { type Props = {
data: { data: {
@@ -207,7 +203,6 @@
); );
let pendingObservationEvents: ConsoleEventInput[] = []; let pendingObservationEvents: ConsoleEventInput[] = [];
let protocolEventSequence = 0; let protocolEventSequence = 0;
let pendingObservedStates: Array<string | null> = [];
let pendingStreamDiagnostics: Diagnostic[] = []; let pendingStreamDiagnostics: Diagnostic[] = [];
let observationFlushHandle: number | null = null; let observationFlushHandle: number | null = null;
let nextReloadToken = 0; let nextReloadToken = 0;
@@ -249,7 +244,9 @@
const diagnostics = $derived( const diagnostics = $derived(
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics), mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
); );
const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading"); const workerState = $derived(
liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"),
);
const workerRunning = $derived(workerState === "running"); const workerRunning = $derived(workerState === "running");
const workerPaused = $derived(workerState === "paused"); const workerPaused = $derived(workerState === "paused");
const composerEditable = $derived(protocolState === "open" && !sending); const composerEditable = $derived(protocolState === "open" && !sending);
@@ -343,7 +340,6 @@
observationFlushHandle = null; observationFlushHandle = null;
} }
pendingObservationEvents = []; pendingObservationEvents = [];
pendingObservedStates = [];
pendingStreamDiagnostics = []; pendingStreamDiagnostics = [];
} }
@@ -359,18 +355,15 @@
function flushObservationBatch() { function flushObservationBatch() {
observationFlushHandle = null; observationFlushHandle = null;
const eventBatch = pendingObservationEvents; const eventBatch = pendingObservationEvents;
const stateBatch = pendingObservedStates;
const diagnosticBatch = pendingStreamDiagnostics; const diagnosticBatch = pendingStreamDiagnostics;
pendingObservationEvents = []; pendingObservationEvents = [];
pendingObservedStates = [];
pendingStreamDiagnostics = []; pendingStreamDiagnostics = [];
if (eventBatch.length > 0) { if (eventBatch.length > 0) {
const latestState = stateBatch.findLast((state) => state !== null);
if (latestState) {
liveWorkerState = latestState;
}
consoleProjection = consoleProjector.append(eventBatch); consoleProjection = consoleProjector.append(eventBatch);
liveWorkerState = consoleProjection.status === "shutdown"
? "shutdown"
: workerStateFromSnapshot(consoleProjection.workerState);
advanceEventObservedAtVersion(); advanceEventObservedAtVersion();
} }
@@ -407,7 +400,6 @@
event: payload, event: payload,
observedAtMs, observedAtMs,
}); });
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
scheduleObservationFlush(); scheduleObservationFlush();
} }
@@ -541,9 +533,42 @@
} }
} }
let nextWorkerCommandId = 1;
function lifecycleMethod(
command: "pause" | "cancel" | "resume" | "compact",
): ProtocolMethod | null {
const state = consoleProjection.workerState;
if (!state) {
sendError = "Worker state snapshot is not available; reconnect before sending control.";
return null;
}
const commandId = Math.max(
nextWorkerCommandId,
state.last_command_id + 1,
);
nextWorkerCommandId = commandId + 1;
const envelope = {
command_id: commandId,
expected_execution_generation: state.execution_generation,
expected_worker_state_revision: state.revision,
};
switch (command) {
case "pause":
return { method: "pause", params: { command: envelope } };
case "cancel":
return { method: "cancel", params: { command: envelope } };
case "resume":
return { method: "resume", params: { command: envelope } };
case "compact":
return { method: "compact", params: { command: envelope } };
}
}
function sendWorkerControl(command: "pause" | "cancel" | "resume") { function sendWorkerControl(command: "pause" | "cancel" | "resume") {
const label = command[0].toUpperCase() + command.slice(1); const label = command[0].toUpperCase() + command.slice(1);
sendControl({ method: command }, label); const method = lifecycleMethod(command);
if (method) sendControl(method, label);
} }
function isEditableTarget(target: EventTarget | null): boolean { function isEditableTarget(target: EventTarget | null): boolean {
@@ -627,8 +652,11 @@
auto_run: true, auto_run: true,
}, },
}; };
case "compact": case "compact": {
return { method: "compact" }; const method = lifecycleMethod("compact");
if (!method) throw new Error("Worker state snapshot is not available");
return method;
}
case "list_rewind_targets": case "list_rewind_targets":
return { method: "list_rewind_targets" }; return { method: "list_rewind_targets" };
case "register_peer": case "register_peer":
@@ -691,7 +719,7 @@
function handleComposerSubmit() { function handleComposerSubmit() {
if (workerRunning) { if (workerRunning) {
sendControl({ method: "cancel" }, "Stop"); sendWorkerControl("cancel");
return; return;
} }
void submitDraft(composerInputElement?.snapshot() ?? draft); void submitDraft(composerInputElement?.snapshot() ?? draft);
@@ -889,18 +917,16 @@
handleComposerSubmit(); handleComposerSubmit();
} }
function workerStateFromProtocolEvent( function workerStateFromSnapshot(
event: PodProtocolEvent, snapshot: ConsoleProjection["workerState"],
): string | null { ): string | null {
switch (event.event) { if (!snapshot) return null;
case "snapshot": return snapshot.state.kind === "idle"
case "status": ? "idle"
return event.data.status; : snapshot.state.state.kind === "run" &&
case "shutdown": snapshot.state.state.state === "paused"
return "shutdown"; ? "paused"
default: : "running";
return null;
}
} }
function connectProtocolTransport( function connectProtocolTransport(
@@ -1620,7 +1646,10 @@
type="button" type="button"
class="secondary-button" class="secondary-button"
disabled={protocolState !== "open"} disabled={protocolState !== "open"}
onclick={() => sendControl({ method: "compact" }, "Compact")} onclick={() => {
const method = lifecycleMethod("compact");
if (method) sendControl(method, "Compact");
}}
> >
Compact Compact
</button> </button>
@@ -4,6 +4,7 @@
import { workerHref } from '$lib/workspace/resource-links'; import { workerHref } from '$lib/workspace/resource-links';
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision'; import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers'; import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types'; import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
import type { PageProps } from './$types'; import type { PageProps } from './$types';
@@ -136,7 +137,7 @@
} }
function workerStatus(worker: Worker): string { function workerStatus(worker: Worker): string {
return worker.state; return liveWorkerState(worker);
} }
function workerProfile(worker: Worker): string { function workerProfile(worker: Worker): string {
@@ -1,5 +1,6 @@
<script lang="ts"> <script lang="ts">
import { workspaceRoute } from '$lib/workspace/api/http'; import { workspaceRoute } from '$lib/workspace/api/http';
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
import type { PageData } from './$types'; import type { PageData } from './$types';
let { data }: { data: PageData } = $props(); let { data }: { data: PageData } = $props();
</script> </script>
@@ -24,7 +25,7 @@
>Open console</a> >Open console</a>
</header> </header>
<dl class="resource-meta"> <dl class="resource-meta">
<dt>Status</dt><dd>{data.worker.state}</dd> <dt>Status</dt><dd>{liveWorkerState(data.worker)}</dd>
<dt>Profile</dt><dd>{data.worker.profile}</dd> <dt>Profile</dt><dd>{data.worker.profile}</dd>
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd> <dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
</dl> </dl>