feat: integrate Worker state authority
This commit is contained in:
@@ -112,8 +112,8 @@ mod tests {
|
||||
async fn encodes_methods_and_decodes_events_above_transport() {
|
||||
let mut socket = TestSocket::default();
|
||||
socket.incoming.push_back(
|
||||
encode_event(&Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
encode_event(&Event::WorkerState {
|
||||
snapshot: WorkerStatus::Idle.into(),
|
||||
})
|
||||
.expect("encode event"),
|
||||
);
|
||||
@@ -132,9 +132,7 @@ mod tests {
|
||||
));
|
||||
assert!(matches!(
|
||||
client.next_event().await,
|
||||
Ok(Some(Event::Status {
|
||||
status: WorkerStatus::Idle
|
||||
}))
|
||||
Ok(Some(Event::WorkerState { .. }))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,8 +101,8 @@ mod tests {
|
||||
));
|
||||
|
||||
peer.send(
|
||||
encode_event(&Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
encode_event(&Event::WorkerState {
|
||||
snapshot: WorkerStatus::Idle.into(),
|
||||
})
|
||||
.expect("encode event"),
|
||||
)
|
||||
@@ -110,9 +110,7 @@ mod tests {
|
||||
.expect("send event");
|
||||
assert!(matches!(
|
||||
client.next_event().await,
|
||||
Ok(Some(Event::Status {
|
||||
status: WorkerStatus::Idle
|
||||
}))
|
||||
Ok(Some(Event::WorkerState { .. }))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,8 +113,8 @@ mod tests {
|
||||
let listener = UnixListener::bind(&socket_path).unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let event = encode_event(&Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
let event = encode_event(&Event::WorkerState {
|
||||
snapshot: WorkerStatus::Idle.into(),
|
||||
})
|
||||
.unwrap();
|
||||
stream.write_all(event.as_bytes()).await.unwrap();
|
||||
@@ -126,12 +126,7 @@ mod tests {
|
||||
.await
|
||||
.expect("client should receive event while alive")
|
||||
.expect("transport should succeed");
|
||||
assert!(matches!(
|
||||
event,
|
||||
Some(Event::Status {
|
||||
status: WorkerStatus::Idle
|
||||
})
|
||||
));
|
||||
assert!(matches!(event, Some(Event::WorkerState { .. })));
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
|
||||
@@ -116,8 +116,8 @@ mod tests {
|
||||
Message::Text(ref text)
|
||||
if matches!(decode_method(text), Ok(Method::Submit { .. }))
|
||||
));
|
||||
let event = encode_event(&Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
let event = encode_event(&Event::WorkerState {
|
||||
snapshot: WorkerStatus::Idle.into(),
|
||||
})
|
||||
.unwrap();
|
||||
socket.send(Message::Text(event.into())).await.unwrap();
|
||||
@@ -134,9 +134,7 @@ mod tests {
|
||||
.expect("send method");
|
||||
assert!(matches!(
|
||||
client.next_event().await,
|
||||
Ok(Some(Event::Status {
|
||||
status: WorkerStatus::Idle
|
||||
}))
|
||||
Ok(Some(Event::WorkerState { .. }))
|
||||
));
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
+321
-51
@@ -85,6 +85,190 @@ impl AuthenticatedInputSource {
|
||||
}
|
||||
}
|
||||
|
||||
/// Immutable identity and revision fence for one state-changing Worker command.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
pub struct WorkerCommandEnvelope {
|
||||
/// Caller-owned sequence. A controller accepts command ids in strictly
|
||||
/// increasing order for one execution generation.
|
||||
pub command_id: u64,
|
||||
pub expected_execution_generation: u64,
|
||||
pub expected_worker_state_revision: u64,
|
||||
}
|
||||
|
||||
impl WorkerCommandEnvelope {
|
||||
pub fn for_snapshot(command_id: u64, snapshot: &WorkerStateSnapshot) -> Self {
|
||||
Self {
|
||||
command_id,
|
||||
expected_execution_generation: snapshot.execution_generation,
|
||||
expected_worker_state_revision: snapshot.revision,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerCommandKind {
|
||||
Resume,
|
||||
Cancel,
|
||||
Pause,
|
||||
Compact,
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerCommandDisposition {
|
||||
Accepted,
|
||||
StaleExecutionGeneration,
|
||||
StaleWorkerStateRevision,
|
||||
StaleCommandId,
|
||||
Conflict,
|
||||
InvalidState,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
pub struct WorkerCommandAcknowledgement {
|
||||
pub command_id: u64,
|
||||
pub command: WorkerCommandKind,
|
||||
pub disposition: WorkerCommandDisposition,
|
||||
/// The complete authoritative state observed after command admission.
|
||||
pub state: WorkerStateSnapshot,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "kind", content = "state", rename_all = "snake_case")]
|
||||
pub enum WorkerState {
|
||||
Idle,
|
||||
Busy(WorkerBusyState),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "kind", content = "state", rename_all = "snake_case")]
|
||||
pub enum WorkerBusyState {
|
||||
Run(WorkerRunState),
|
||||
Maintenance(WorkerMaintenanceState),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerRunState {
|
||||
Running,
|
||||
Pausing,
|
||||
Paused,
|
||||
Cancelling,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerMaintenanceState {
|
||||
Compacting,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
pub struct WorkerStateSnapshot {
|
||||
pub execution_generation: u64,
|
||||
pub revision: u64,
|
||||
/// Highest lifecycle command id observed by this controller generation.
|
||||
pub last_command_id: u64,
|
||||
pub state: WorkerState,
|
||||
}
|
||||
|
||||
impl WorkerStateSnapshot {
|
||||
pub fn initial(execution_generation: u64) -> Self {
|
||||
Self {
|
||||
execution_generation,
|
||||
revision: 0,
|
||||
last_command_id: 0,
|
||||
state: WorkerState::Idle,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compatibility projection for Runtime catalog lifecycle. This value is
|
||||
/// never command-admission authority and cannot produce `Stopped`.
|
||||
pub fn catalog_status(&self) -> WorkerStatus {
|
||||
match self.state {
|
||||
WorkerState::Idle => WorkerStatus::Idle,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused,
|
||||
WorkerState::Busy(WorkerBusyState::Run(_))
|
||||
| WorkerState::Busy(WorkerBusyState::Maintenance(_)) => WorkerStatus::Running,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum WorkerStateSnapshotApply {
|
||||
Applied,
|
||||
Duplicate,
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct WorkerStateSnapshotConflict {
|
||||
pub execution_generation: u64,
|
||||
pub revision: u64,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for WorkerStateSnapshotConflict {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"conflicting worker state snapshots at generation {} revision {}",
|
||||
self.execution_generation, self.revision
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for WorkerStateSnapshotConflict {}
|
||||
|
||||
pub fn apply_worker_state_snapshot(
|
||||
current: &mut WorkerStateSnapshot,
|
||||
incoming: &WorkerStateSnapshot,
|
||||
) -> Result<WorkerStateSnapshotApply, WorkerStateSnapshotConflict> {
|
||||
use std::cmp::Ordering;
|
||||
|
||||
let ordering = (incoming.execution_generation, incoming.revision)
|
||||
.cmp(&(current.execution_generation, current.revision));
|
||||
match ordering {
|
||||
Ordering::Greater => {
|
||||
*current = incoming.clone();
|
||||
Ok(WorkerStateSnapshotApply::Applied)
|
||||
}
|
||||
Ordering::Less => Ok(WorkerStateSnapshotApply::Stale),
|
||||
Ordering::Equal if incoming == current => Ok(WorkerStateSnapshotApply::Duplicate),
|
||||
Ordering::Equal => Err(WorkerStateSnapshotConflict {
|
||||
execution_generation: incoming.execution_generation,
|
||||
revision: incoming.revision,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
impl From<WorkerStatus> for WorkerStateSnapshot {
|
||||
fn from(status: WorkerStatus) -> Self {
|
||||
let state = match status {
|
||||
WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
|
||||
WorkerStatus::Running => {
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||
}
|
||||
WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
|
||||
};
|
||||
Self {
|
||||
execution_generation: 1,
|
||||
revision: 0,
|
||||
last_command_id: 0,
|
||||
state,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "method", content = "params", rename_all = "snake_case")]
|
||||
@@ -149,20 +333,28 @@ pub enum Method {
|
||||
expected_revision: u64,
|
||||
expected_head_id: String,
|
||||
},
|
||||
Resume,
|
||||
Cancel,
|
||||
Resume {
|
||||
command: WorkerCommandEnvelope,
|
||||
},
|
||||
Cancel {
|
||||
command: WorkerCommandEnvelope,
|
||||
},
|
||||
/// Stop the in-flight turn and transition to `Paused`.
|
||||
///
|
||||
/// Unlike `Cancel` (which discards and returns to `Idle`), a paused
|
||||
/// Worker can resume the interrupted work via `Resume`, or accept a
|
||||
/// fresh `Submit` (orphan `tool_use` items are closed with a
|
||||
/// synthetic tool result before the new user message is appended).
|
||||
Pause,
|
||||
Pause {
|
||||
command: WorkerCommandEnvelope,
|
||||
},
|
||||
/// Request an explicit compaction while the Worker is otherwise idle.
|
||||
///
|
||||
/// This is a typed control method: clients must not send `compact` as a
|
||||
/// `Method::Submit` user message.
|
||||
Compact,
|
||||
Compact {
|
||||
command: WorkerCommandEnvelope,
|
||||
},
|
||||
/// Ask the Worker to list valid rewind targets from its authoritative session log.
|
||||
ListRewindTargets,
|
||||
/// Truncate the current session back to the selected rewind target and
|
||||
@@ -171,7 +363,9 @@ pub enum Method {
|
||||
target: RewindTargetId,
|
||||
expected_head_entries: usize,
|
||||
},
|
||||
Shutdown,
|
||||
Shutdown {
|
||||
command: WorkerCommandEnvelope,
|
||||
},
|
||||
/// Request a list of completion candidates from the Worker.
|
||||
///
|
||||
/// Reply is sent on the same socket as `Event::Completions` (not
|
||||
@@ -938,8 +1132,9 @@ pub enum Event {
|
||||
Snapshot {
|
||||
session: SessionSnapshot,
|
||||
greeting: Greeting,
|
||||
#[serde(default)]
|
||||
status: WorkerStatus,
|
||||
/// Full revisioned live execution state. `Stopped` remains Runtime
|
||||
/// catalog authority and is deliberately not represented here.
|
||||
state: WorkerStateSnapshot,
|
||||
/// Unfinished model output that has already streamed in the current
|
||||
/// run but is not yet represented by committed snapshot entries.
|
||||
#[serde(default, skip_serializing_if = "InFlightSnapshot::is_empty")]
|
||||
@@ -976,8 +1171,11 @@ pub enum Event {
|
||||
},
|
||||
/// Current Worker controller status. Broadcast on every controller-level
|
||||
/// transition and included in `History` snapshots for late attach.
|
||||
Status {
|
||||
status: WorkerStatus,
|
||||
WorkerState {
|
||||
snapshot: WorkerStateSnapshot,
|
||||
},
|
||||
CommandAcknowledged {
|
||||
acknowledgement: WorkerCommandAcknowledgement,
|
||||
},
|
||||
/// Bounded, provider-owned command telemetry for the live Console. This is
|
||||
/// intentionally not a history entry and is reconstructed from
|
||||
@@ -1424,6 +1622,58 @@ pub enum Permission {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn worker_state_snapshot_apply_is_monotonic_and_detects_conflicts() {
|
||||
let mut current = WorkerStateSnapshot::initial(4);
|
||||
let mut newer = current.clone();
|
||||
newer.revision = 1;
|
||||
newer.state = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running));
|
||||
|
||||
assert_eq!(
|
||||
apply_worker_state_snapshot(&mut current, &newer),
|
||||
Ok(WorkerStateSnapshotApply::Applied)
|
||||
);
|
||||
assert_eq!(
|
||||
apply_worker_state_snapshot(&mut current, &newer),
|
||||
Ok(WorkerStateSnapshotApply::Duplicate)
|
||||
);
|
||||
|
||||
let stale_revision = WorkerStateSnapshot::initial(4);
|
||||
assert_eq!(
|
||||
apply_worker_state_snapshot(&mut current, &stale_revision),
|
||||
Ok(WorkerStateSnapshotApply::Stale)
|
||||
);
|
||||
let stale_generation = WorkerStateSnapshot {
|
||||
execution_generation: 3,
|
||||
revision: u64::MAX,
|
||||
..newer.clone()
|
||||
};
|
||||
assert_eq!(
|
||||
apply_worker_state_snapshot(&mut current, &stale_generation),
|
||||
Ok(WorkerStateSnapshotApply::Stale)
|
||||
);
|
||||
|
||||
let conflicting = WorkerStateSnapshot {
|
||||
state: WorkerState::Idle,
|
||||
..newer.clone()
|
||||
};
|
||||
assert_eq!(
|
||||
apply_worker_state_snapshot(&mut current, &conflicting),
|
||||
Err(WorkerStateSnapshotConflict {
|
||||
execution_generation: 4,
|
||||
revision: 1,
|
||||
})
|
||||
);
|
||||
assert_eq!(current, newer);
|
||||
|
||||
let next_generation = WorkerStateSnapshot::initial(5);
|
||||
assert_eq!(
|
||||
apply_worker_state_snapshot(&mut current, &next_generation),
|
||||
Ok(WorkerStateSnapshotApply::Applied)
|
||||
);
|
||||
assert_eq!(current, next_generation);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_submit_json_roundtrip_and_run_is_rejected() {
|
||||
let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#;
|
||||
@@ -1612,28 +1862,39 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_without_params() {
|
||||
let json = r#"{"method":"resume"}"#;
|
||||
let method: Method = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(method, Method::Resume));
|
||||
fn lifecycle_method_without_command_fails_closed() {
|
||||
let error = serde_json::from_str::<Method>(r#"{"method":"resume"}"#).unwrap_err();
|
||||
assert!(error.to_string().contains("params"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_pause_roundtrip() {
|
||||
let json = r#"{"method":"pause"}"#;
|
||||
let method: Method = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(method, Method::Pause));
|
||||
let serialized = serde_json::to_string(&method).unwrap();
|
||||
assert_eq!(serialized, json);
|
||||
fn lifecycle_methods_roundtrip_with_fences() {
|
||||
for method in [
|
||||
Method::Pause {
|
||||
command: WorkerCommandEnvelope {
|
||||
command_id: 11,
|
||||
expected_execution_generation: 4,
|
||||
expected_worker_state_revision: 8,
|
||||
},
|
||||
},
|
||||
Method::Compact {
|
||||
command: WorkerCommandEnvelope {
|
||||
command_id: 12,
|
||||
expected_execution_generation: 4,
|
||||
expected_worker_state_revision: 9,
|
||||
},
|
||||
},
|
||||
] {
|
||||
let json = serde_json::to_string(&method).unwrap();
|
||||
let decoded: Method = serde_json::from_str(&json).unwrap();
|
||||
match decoded {
|
||||
Method::Pause { command } | Method::Compact { command } => {
|
||||
assert_eq!(command.expected_execution_generation, 4);
|
||||
assert!(command.command_id >= 11);
|
||||
}
|
||||
other => panic!("unexpected lifecycle method: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_compact_roundtrip() {
|
||||
let json = r#"{"method":"compact"}"#;
|
||||
let method: Method = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(method, Method::Compact));
|
||||
let serialized = serde_json::to_string(&method).unwrap();
|
||||
assert_eq!(serialized, json);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1902,7 +2163,7 @@ mod tests {
|
||||
context_window: 200_000,
|
||||
context_tokens: 42_000,
|
||||
},
|
||||
status: WorkerStatus::Paused,
|
||||
state: WorkerStatus::Paused.into(),
|
||||
in_flight: InFlightSnapshot::default(),
|
||||
internal_workers: Vec::new(),
|
||||
};
|
||||
@@ -1919,12 +2180,13 @@ mod tests {
|
||||
assert_eq!(parsed["data"]["greeting"]["tools"][0], "Read");
|
||||
assert_eq!(parsed["data"]["greeting"]["context_window"], 200_000);
|
||||
assert_eq!(parsed["data"]["greeting"]["context_tokens"], 42_000);
|
||||
assert_eq!(parsed["data"]["status"], "paused");
|
||||
assert_eq!(parsed["data"]["state"]["state"]["kind"], "busy");
|
||||
assert_eq!(parsed["data"]["state"]["state"]["state"]["state"], "paused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_snapshot_in_flight_roundtrip_and_default() {
|
||||
let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"status":"running"}}"#;
|
||||
let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"state":{"execution_generation":1,"revision":1,"last_command_id":0,"state":{"kind":"busy","state":{"kind":"run","state":"running"}}}}}"#;
|
||||
let decoded: Event = serde_json::from_str(inbound).unwrap();
|
||||
match decoded {
|
||||
Event::Snapshot { in_flight, .. } => assert!(in_flight.is_empty()),
|
||||
@@ -1946,7 +2208,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Running,
|
||||
state: WorkerStatus::Running.into(),
|
||||
in_flight: InFlightSnapshot {
|
||||
blocks: vec![
|
||||
InFlightBlock::Text {
|
||||
@@ -2034,20 +2296,32 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_status_format() {
|
||||
let event = Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
fn event_worker_state_format() {
|
||||
let event = Event::WorkerState {
|
||||
snapshot: WorkerStateSnapshot {
|
||||
execution_generation: 7,
|
||||
revision: 3,
|
||||
last_command_id: 9,
|
||||
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||
},
|
||||
};
|
||||
let json = serde_json::to_string(&event).unwrap();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed["event"], "status");
|
||||
assert_eq!(parsed["data"]["status"], "running");
|
||||
assert_eq!(parsed["event"], "worker_state");
|
||||
assert_eq!(parsed["data"]["snapshot"]["execution_generation"], 7);
|
||||
assert_eq!(parsed["data"]["snapshot"]["revision"], 3);
|
||||
assert_eq!(parsed["data"]["snapshot"]["state"]["kind"], "busy");
|
||||
|
||||
let decoded: Event = serde_json::from_str(&json).unwrap();
|
||||
assert!(matches!(
|
||||
decoded,
|
||||
Event::Status {
|
||||
status: WorkerStatus::Running
|
||||
Event::WorkerState {
|
||||
snapshot: WorkerStateSnapshot {
|
||||
execution_generation: 7,
|
||||
revision: 3,
|
||||
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||
..
|
||||
}
|
||||
}
|
||||
));
|
||||
}
|
||||
@@ -2088,19 +2362,10 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_snapshot_without_status_defaults_to_idle() {
|
||||
fn event_snapshot_without_worker_state_fails_closed() {
|
||||
let json = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#;
|
||||
let decoded: Event = serde_json::from_str(json).unwrap();
|
||||
match decoded {
|
||||
Event::Snapshot {
|
||||
status, greeting, ..
|
||||
} => {
|
||||
assert_eq!(status, WorkerStatus::Idle);
|
||||
assert_eq!(greeting.context_window, 0);
|
||||
assert_eq!(greeting.context_tokens, 0);
|
||||
}
|
||||
other => panic!("expected Snapshot, got {other:?}"),
|
||||
}
|
||||
let error = serde_json::from_str::<Event>(json).unwrap_err();
|
||||
assert!(error.to_string().contains("state"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2513,7 +2778,12 @@ mod tests {
|
||||
"scope_summary": "scope",
|
||||
"tools": []
|
||||
},
|
||||
"status": "idle"
|
||||
"state": {
|
||||
"execution_generation": 1,
|
||||
"revision": 0,
|
||||
"last_command_id": 0,
|
||||
"state": { "kind": "idle" }
|
||||
}
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
@@ -573,6 +573,11 @@ pub struct SubscriptionWorker {
|
||||
pub resource_key: Option<String>,
|
||||
/// Producer-owned monotonic revision for this Worker subject.
|
||||
pub subject_revision: u64,
|
||||
/// Latest revisioned foreground state observed from the Worker. This remains
|
||||
/// absent until an authoritative Worker snapshot/event has been applied.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub worker_state: Option<crate::WorkerStateSnapshot>,
|
||||
/// Runtime catalog lifecycle compatibility projection; not foreground-state authority.
|
||||
pub state: SubscriptionWorkerState,
|
||||
#[serde(default)]
|
||||
pub has_running_internal_workers: bool,
|
||||
@@ -874,6 +879,7 @@ mod tests {
|
||||
runtime_id: None,
|
||||
resource_key: None,
|
||||
subject_revision: 0,
|
||||
worker_state: None,
|
||||
state: SubscriptionWorkerState::Idle,
|
||||
has_running_internal_workers: false,
|
||||
workspace_id: Some("workspace-1".to_string()),
|
||||
|
||||
@@ -12,7 +12,10 @@ use crate::{
|
||||
RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart,
|
||||
SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry,
|
||||
SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition,
|
||||
TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, WorkerStatus,
|
||||
TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerBusyState,
|
||||
WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope,
|
||||
WorkerCommandKind, WorkerEvent, WorkerMaintenanceState, WorkerRunState, WorkerState,
|
||||
WorkerStateSnapshot, WorkerStatus,
|
||||
subscription::{
|
||||
EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame,
|
||||
SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest,
|
||||
@@ -46,6 +49,15 @@ pub fn generated_protocol_types() -> String {
|
||||
push_decl::<AlertSource>(&cfg, &mut output);
|
||||
push_decl::<CompletionKind>(&cfg, &mut output);
|
||||
push_decl::<WorkerStatus>(&cfg, &mut output);
|
||||
push_decl::<WorkerCommandEnvelope>(&cfg, &mut output);
|
||||
push_decl::<WorkerCommandKind>(&cfg, &mut output);
|
||||
push_decl::<WorkerCommandDisposition>(&cfg, &mut output);
|
||||
push_decl::<WorkerCommandAcknowledgement>(&cfg, &mut output);
|
||||
push_decl::<WorkerRunState>(&cfg, &mut output);
|
||||
push_decl::<WorkerMaintenanceState>(&cfg, &mut output);
|
||||
push_decl::<WorkerBusyState>(&cfg, &mut output);
|
||||
push_decl::<WorkerState>(&cfg, &mut output);
|
||||
push_decl::<WorkerStateSnapshot>(&cfg, &mut output);
|
||||
push_decl::<TurnResult>(&cfg, &mut output);
|
||||
push_decl::<InvokeKind>(&cfg, &mut output);
|
||||
push_decl::<RunResult>(&cfg, &mut output);
|
||||
|
||||
@@ -318,7 +318,11 @@ impl StandaloneHost {
|
||||
}
|
||||
|
||||
pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> {
|
||||
let _ = self.handle.send(Method::Shutdown).await;
|
||||
let command = protocol::WorkerCommandEnvelope::for_snapshot(
|
||||
u64::MAX,
|
||||
&self.handle.shared_state.snapshot(),
|
||||
);
|
||||
let _ = self.handle.send(Method::Shutdown { command }).await;
|
||||
let Some(shutdown) = self.shutdown.take() else {
|
||||
self.retain_lease();
|
||||
return Err(StandaloneShutdownError::ConfirmationLost);
|
||||
@@ -500,7 +504,11 @@ fn active_pointer(
|
||||
}
|
||||
|
||||
async fn stop_started_worker(started: BootstrappedWorker) {
|
||||
let _ = started.handle.send(Method::Shutdown).await;
|
||||
let command = protocol::WorkerCommandEnvelope::for_snapshot(
|
||||
u64::MAX,
|
||||
&started.handle.shared_state.snapshot(),
|
||||
);
|
||||
let _ = started.handle.send(Method::Shutdown { command }).await;
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), started.shutdown).await;
|
||||
}
|
||||
|
||||
|
||||
+158
-38
@@ -5,7 +5,7 @@ use std::time::{Duration, Instant};
|
||||
use protocol::{
|
||||
AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, InFlightBlock,
|
||||
InFlightSnapshot, InFlightToolCallState, InternalWorkerRef, InternalWorkerSnapshot, Method,
|
||||
RewindTarget, RunResult, Segment, WorkerStatus,
|
||||
RewindTarget, RunResult, Segment, WorkerCommandEnvelope, WorkerStateSnapshot, WorkerStatus,
|
||||
};
|
||||
|
||||
use crate::block::{
|
||||
@@ -225,8 +225,10 @@ pub struct WorkerViewTab {
|
||||
pub struct App {
|
||||
pub worker_name: String,
|
||||
pub connected: bool,
|
||||
/// Last controller status reported by the Worker. Drives the status line
|
||||
/// and Ctrl-key routing; do not infer this solely from replayed history.
|
||||
/// Latest authoritative revisioned live execution state.
|
||||
pub worker_state: WorkerStateSnapshot,
|
||||
next_command_id: u64,
|
||||
/// Derived Runtime-catalog compatibility projection used by existing UI.
|
||||
pub worker_status: WorkerStatus,
|
||||
/// True while the Worker is in `WorkerStatus::Running`.
|
||||
pub running: bool,
|
||||
@@ -337,6 +339,8 @@ impl App {
|
||||
Self {
|
||||
worker_name,
|
||||
connected: false,
|
||||
worker_state: WorkerStateSnapshot::initial(1),
|
||||
next_command_id: 1,
|
||||
worker_status: WorkerStatus::Idle,
|
||||
running: false,
|
||||
paused: false,
|
||||
@@ -745,7 +749,8 @@ impl App {
|
||||
if self.paused {
|
||||
self.input_history.cancel_browse();
|
||||
self.input.clear();
|
||||
return Some(Method::Resume);
|
||||
let command = self.next_command_envelope();
|
||||
return Some(Method::Resume { command });
|
||||
}
|
||||
return None;
|
||||
}
|
||||
@@ -1114,6 +1119,31 @@ impl App {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_command_envelope(&mut self) -> WorkerCommandEnvelope {
|
||||
let command_id = self
|
||||
.next_command_id
|
||||
.max(self.worker_state.last_command_id.saturating_add(1));
|
||||
let command = WorkerCommandEnvelope::for_snapshot(command_id, &self.worker_state);
|
||||
self.next_command_id = command_id.saturating_add(1);
|
||||
command
|
||||
}
|
||||
|
||||
fn apply_worker_state_snapshot(&mut self, snapshot: &WorkerStateSnapshot) {
|
||||
match protocol::apply_worker_state_snapshot(&mut self.worker_state, snapshot) {
|
||||
Ok(protocol::WorkerStateSnapshotApply::Applied) => {
|
||||
self.set_worker_status(self.worker_state.catalog_status());
|
||||
}
|
||||
Ok(
|
||||
protocol::WorkerStateSnapshotApply::Duplicate
|
||||
| protocol::WorkerStateSnapshotApply::Stale,
|
||||
) => {}
|
||||
Err(error) => self.handle_error(
|
||||
ErrorCode::Internal,
|
||||
format!("worker state stream rejected: {error}"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn handle_worker_event(&mut self, event: Event) -> Option<Method> {
|
||||
if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) {
|
||||
return None;
|
||||
@@ -1150,18 +1180,14 @@ impl App {
|
||||
self.assistant_streaming = false;
|
||||
}
|
||||
Event::TurnStart { .. } => {
|
||||
self.set_worker_status(WorkerStatus::Running);
|
||||
self.run_requests += 1;
|
||||
self.current_tool = None;
|
||||
self.latest_llm_wait_event = None;
|
||||
self.assistant_streaming = false;
|
||||
}
|
||||
Event::InvokeStart { .. } => {
|
||||
self.set_worker_status(WorkerStatus::Running);
|
||||
}
|
||||
Event::InvokeStart { .. } => {}
|
||||
// UI consumers of per-attempt LlmCall semantics remain out of scope;
|
||||
// the run-level status starts at InvokeStart and TurnStart counts each
|
||||
// LLM request within that run.
|
||||
// authoritative run state comes only from WorkerStateSnapshot.
|
||||
Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => {
|
||||
self.latest_llm_wait_event = None;
|
||||
}
|
||||
@@ -1368,12 +1394,7 @@ impl App {
|
||||
output_tokens: self.run_output_tokens,
|
||||
});
|
||||
self.pending_submit_rollback = None;
|
||||
self.reset_run_state(match result {
|
||||
RunResult::Paused => WorkerStatus::Paused,
|
||||
RunResult::Finished | RunResult::LimitReached | RunResult::RolledBack => {
|
||||
WorkerStatus::Idle
|
||||
}
|
||||
});
|
||||
self.reset_run_state();
|
||||
}
|
||||
}
|
||||
Event::CompactStart { .. } => {
|
||||
@@ -1443,7 +1464,7 @@ impl App {
|
||||
Event::Snapshot {
|
||||
session,
|
||||
greeting,
|
||||
status,
|
||||
state,
|
||||
in_flight,
|
||||
internal_workers,
|
||||
} => {
|
||||
@@ -1451,7 +1472,7 @@ impl App {
|
||||
self.pending_submissions = session.pending_submissions.clone();
|
||||
self.restore_snapshot(&session, greeting, in_flight);
|
||||
self.replace_internal_worker_snapshots(internal_workers);
|
||||
self.set_worker_status(status);
|
||||
self.apply_worker_state_snapshot(&state);
|
||||
}
|
||||
Event::InternalWorker {
|
||||
worker,
|
||||
@@ -1461,9 +1482,12 @@ impl App {
|
||||
Event::InternalWorkerRemoved { worker, revision } => {
|
||||
self.remove_internal_worker(worker, revision)
|
||||
}
|
||||
Event::Status { status } => {
|
||||
Event::WorkerState { snapshot } => {
|
||||
self.rewind_refresh_fence = false;
|
||||
self.set_worker_status(status);
|
||||
self.apply_worker_state_snapshot(&snapshot);
|
||||
}
|
||||
Event::CommandAcknowledged { acknowledgement } => {
|
||||
self.apply_worker_state_snapshot(&acknowledgement.state);
|
||||
}
|
||||
// Command telemetry is an operational Web Console surface. The
|
||||
// TUI continues to render the final Bash ToolResult from history.
|
||||
@@ -1503,7 +1527,7 @@ impl App {
|
||||
};
|
||||
self.completion = None;
|
||||
self.close_rewind_picker();
|
||||
self.reset_run_state(self.worker_status);
|
||||
self.reset_run_state();
|
||||
let mut message = if restored_composer {
|
||||
format!(
|
||||
"Rewound session: discarded {} log entries; restored selected input to composer.",
|
||||
@@ -1551,8 +1575,7 @@ impl App {
|
||||
None
|
||||
}
|
||||
|
||||
fn reset_run_state(&mut self, status: WorkerStatus) {
|
||||
self.set_worker_status(status);
|
||||
fn reset_run_state(&mut self) {
|
||||
self.run_requests = 0;
|
||||
self.run_upload_tokens = 0;
|
||||
self.run_output_tokens = 0;
|
||||
@@ -1582,7 +1605,7 @@ impl App {
|
||||
"Rolled back empty assistant turn; no local submitted input was available to restore."
|
||||
.to_owned()
|
||||
};
|
||||
self.reset_run_state(WorkerStatus::Idle);
|
||||
self.reset_run_state();
|
||||
self.blocks.push(Block::Alert {
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
@@ -2026,12 +2049,18 @@ impl App {
|
||||
self.input_mode = CommandInputMode::Composer;
|
||||
self.command_completion_selected = None;
|
||||
}
|
||||
if let Some(Method::ListRewindTargets) = result.method.as_ref() {
|
||||
let mut method = result.method;
|
||||
if let Some(Method::Compact { .. }) = method {
|
||||
method = Some(Method::Compact {
|
||||
command: self.next_command_envelope(),
|
||||
});
|
||||
}
|
||||
if let Some(Method::ListRewindTargets) = method.as_ref() {
|
||||
self.completion = None;
|
||||
self.rewind_picker = None;
|
||||
self.rewind_request_pending = true;
|
||||
}
|
||||
result.method
|
||||
method
|
||||
}
|
||||
|
||||
fn push_command_diagnostic(&mut self, message: impl Into<String>) {
|
||||
@@ -2761,8 +2790,8 @@ mod rewind_refresh_tests {
|
||||
});
|
||||
assert!(!blocks_contain(&app, "stale tail after rewind"));
|
||||
|
||||
app.handle_worker_event(Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
app.handle_worker_event(Event::WorkerState {
|
||||
snapshot: WorkerStatus::Idle.into(),
|
||||
});
|
||||
app.handle_worker_event(Event::TextDelta {
|
||||
text: "new live tail after status".into(),
|
||||
@@ -3478,7 +3507,7 @@ mod completion_flow_tests {
|
||||
let mut app = App::new("test".into());
|
||||
app.set_worker_status(WorkerStatus::Paused);
|
||||
|
||||
assert!(matches!(app.submit_input(), Some(Method::Resume)));
|
||||
assert!(matches!(app.submit_input(), Some(Method::Resume { .. })));
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
}
|
||||
|
||||
@@ -3533,7 +3562,7 @@ mod completion_flow_tests {
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: public_session(vec![session_start_value]),
|
||||
status: WorkerStatus::Running,
|
||||
state: test_worker_state(WorkerStatus::Running),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
@@ -3544,6 +3573,90 @@ mod completion_flow_tests {
|
||||
assert!(matches!(app.blocks.first(), Some(Block::Greeting(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn occurrence_events_do_not_infer_foreground_worker_state() {
|
||||
let mut app = App::new("test".into());
|
||||
app.handle_worker_event(Event::TurnStart { turn: 1 });
|
||||
app.handle_worker_event(Event::InvokeStart {
|
||||
kind: protocol::InvokeKind::UserSend,
|
||||
});
|
||||
app.handle_worker_event(Event::RunEnd {
|
||||
result: RunResult::Paused,
|
||||
});
|
||||
assert_eq!(app.worker_state.state, protocol::WorkerState::Idle);
|
||||
assert_eq!(app.worker_status, WorkerStatus::Idle);
|
||||
|
||||
let running = WorkerStateSnapshot {
|
||||
execution_generation: 1,
|
||||
revision: 1,
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)),
|
||||
last_command_id: 0,
|
||||
};
|
||||
app.handle_worker_event(Event::WorkerState {
|
||||
snapshot: running.clone(),
|
||||
});
|
||||
app.handle_worker_event(Event::RunEnd {
|
||||
result: RunResult::Finished,
|
||||
});
|
||||
assert_eq!(app.worker_state, running);
|
||||
assert_eq!(app.worker_status, WorkerStatus::Running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_state_events_and_acknowledgements_share_monotonic_application() {
|
||||
let mut app = App::new("test".into());
|
||||
let running = WorkerStateSnapshot {
|
||||
execution_generation: 4,
|
||||
revision: 3,
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)),
|
||||
last_command_id: 2,
|
||||
};
|
||||
app.handle_worker_event(Event::WorkerState {
|
||||
snapshot: running.clone(),
|
||||
});
|
||||
app.handle_worker_event(Event::WorkerState {
|
||||
snapshot: WorkerStateSnapshot {
|
||||
revision: 2,
|
||||
state: protocol::WorkerState::Idle,
|
||||
..running.clone()
|
||||
},
|
||||
});
|
||||
assert_eq!(app.worker_state, running);
|
||||
|
||||
let paused = WorkerStateSnapshot {
|
||||
revision: 4,
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Paused,
|
||||
)),
|
||||
last_command_id: 3,
|
||||
..running.clone()
|
||||
};
|
||||
app.handle_worker_event(Event::CommandAcknowledged {
|
||||
acknowledgement: protocol::WorkerCommandAcknowledgement {
|
||||
command_id: 3,
|
||||
command: protocol::WorkerCommandKind::Pause,
|
||||
disposition: protocol::WorkerCommandDisposition::Accepted,
|
||||
state: paused.clone(),
|
||||
},
|
||||
});
|
||||
assert_eq!(app.worker_state, paused);
|
||||
|
||||
app.handle_worker_event(Event::WorkerState {
|
||||
snapshot: WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Idle,
|
||||
..paused.clone()
|
||||
},
|
||||
});
|
||||
assert_eq!(app.worker_state, paused);
|
||||
assert!(app.run_error_messages.iter().any(|message| {
|
||||
message.contains("conflicting worker state snapshots at generation 4 revision 4")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_replaces_live_error_with_one_durable_run_error_block() {
|
||||
let mut app = App::new("test".into());
|
||||
@@ -3551,8 +3664,8 @@ mod completion_flow_tests {
|
||||
code: ErrorCode::ProviderError,
|
||||
message: "provider unavailable".into(),
|
||||
});
|
||||
app.handle_worker_event(Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
app.handle_worker_event(Event::WorkerState {
|
||||
snapshot: WorkerStatus::Idle.into(),
|
||||
});
|
||||
|
||||
let live_errors = app
|
||||
@@ -3577,7 +3690,7 @@ mod completion_flow_tests {
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: public_session(vec![serde_json::to_value(run_errored).unwrap()]),
|
||||
status: WorkerStatus::Idle,
|
||||
state: test_worker_state(WorkerStatus::Idle),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
@@ -3641,7 +3754,7 @@ mod completion_flow_tests {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Running,
|
||||
state: test_worker_state(WorkerStatus::Running),
|
||||
in_flight: InFlightSnapshot {
|
||||
blocks: vec![
|
||||
InFlightBlock::Thinking {
|
||||
@@ -3968,7 +4081,7 @@ mod completion_flow_tests {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: test_worker_state(WorkerStatus::Idle),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
@@ -4020,7 +4133,7 @@ mod completion_flow_tests {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: test_worker_state(WorkerStatus::Idle),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: vec![InternalWorkerSnapshot {
|
||||
worker: InternalWorkerRef {
|
||||
@@ -4168,6 +4281,13 @@ mod completion_flow_tests {
|
||||
.count()
|
||||
}
|
||||
|
||||
fn test_worker_state(status: WorkerStatus) -> WorkerStateSnapshot {
|
||||
let mut snapshot = WorkerStateSnapshot::from(status);
|
||||
snapshot.execution_generation = 1;
|
||||
snapshot.revision = 1;
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn test_greeting() -> protocol::Greeting {
|
||||
protocol::Greeting {
|
||||
worker_name: "test".into(),
|
||||
@@ -4194,7 +4314,7 @@ mod completion_flow_tests {
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting,
|
||||
status: WorkerStatus::Idle,
|
||||
state: test_worker_state(WorkerStatus::Idle),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
@@ -4393,7 +4513,7 @@ mod completion_flow_tests {
|
||||
app.handle_worker_event(Event::Snapshot {
|
||||
greeting: test_greeting(),
|
||||
session: public_session(assistant_item_entries),
|
||||
status: WorkerStatus::Running,
|
||||
state: test_worker_state(WorkerStatus::Running),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
|
||||
@@ -321,7 +321,7 @@ fn row_line(
|
||||
Span::raw(" "),
|
||||
Span::styled(
|
||||
pad_column(&worker_state(worker), widths.state),
|
||||
state_style(worker.state.as_str()),
|
||||
state_style(worker_state_label(worker)),
|
||||
),
|
||||
Span::raw(" "),
|
||||
Span::styled(
|
||||
@@ -341,8 +341,20 @@ fn worker_name(worker: &BackendWorkerSummary) -> &str {
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_state_label(worker: &BackendWorkerSummary) -> &str {
|
||||
match worker.worker_state.as_ref().map(|state| &state.state) {
|
||||
Some(protocol::WorkerState::Idle) => "idle",
|
||||
Some(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Paused,
|
||||
))) => "paused",
|
||||
Some(protocol::WorkerState::Busy(_)) => "running",
|
||||
None if worker.state == "stopped" => "stopped",
|
||||
None => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_state(worker: &BackendWorkerSummary) -> String {
|
||||
format!("[{}]", worker.state)
|
||||
format!("[{}]", worker_state_label(worker))
|
||||
}
|
||||
|
||||
fn text_width(value: &str) -> usize {
|
||||
@@ -413,7 +425,15 @@ mod tests {
|
||||
identity: "ws".to_string(),
|
||||
workspace_id: Some("ws".to_string()),
|
||||
},
|
||||
state: "running".to_string(),
|
||||
state: "idle".to_string(),
|
||||
worker_state: Some(protocol::WorkerStateSnapshot {
|
||||
execution_generation: 1,
|
||||
revision: 1,
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)),
|
||||
last_command_id: 0,
|
||||
}),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: String::new(),
|
||||
@@ -450,6 +470,7 @@ mod tests {
|
||||
worker.display_name = "Coder".to_string();
|
||||
worker.label = "Coder · T-585".to_string();
|
||||
worker.state = "stopped".to_string();
|
||||
worker.worker_state = None;
|
||||
worker.working_directory = Some(
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"working_directory_id": "001a06a9f0202000000",
|
||||
@@ -478,12 +499,19 @@ mod tests {
|
||||
short.label = "Coder".to_string();
|
||||
short.display_name = short.label.clone();
|
||||
short.state = "idle".to_string();
|
||||
short.worker_state = Some(protocol::WorkerStateSnapshot {
|
||||
execution_generation: 1,
|
||||
revision: 2,
|
||||
state: protocol::WorkerState::Idle,
|
||||
last_command_id: 0,
|
||||
});
|
||||
|
||||
let mut long = worker("runtime-a", "worker-b", None);
|
||||
long.resource_key = "W-100".to_string();
|
||||
long.label = "Longer worker · T-9".to_string();
|
||||
long.display_name = long.label.clone();
|
||||
long.state = "stopped".to_string();
|
||||
long.worker_state = None;
|
||||
|
||||
for worker in [&mut short, &mut long] {
|
||||
worker.working_directory = Some(
|
||||
|
||||
@@ -409,7 +409,12 @@ fn compact_command(invocation: CommandInvocation<'_>) -> CommandExecution {
|
||||
let _ = invocation.environment;
|
||||
let _ = invocation.args.raw();
|
||||
CommandExecution {
|
||||
method: Some(Method::Compact),
|
||||
method: Some(Method::Compact {
|
||||
command: protocol::WorkerCommandEnvelope::for_snapshot(
|
||||
0,
|
||||
&protocol::WorkerStateSnapshot::initial(1),
|
||||
),
|
||||
}),
|
||||
diagnostics: vec![CommandDiagnostic::new("compact requested")],
|
||||
exit_command_mode: true,
|
||||
clear_input: true,
|
||||
@@ -483,7 +488,7 @@ mod tests {
|
||||
fn compact_command_returns_compact_method_not_run() {
|
||||
let registry = CommandRegistry::builtins();
|
||||
let result = registry.dispatch("compact", &env());
|
||||
assert!(matches!(result.method, Some(Method::Compact)));
|
||||
assert!(matches!(result.method, Some(Method::Compact { .. })));
|
||||
assert!(result.exit_command_mode);
|
||||
assert!(result.clear_input);
|
||||
assert!(result.diagnostics[0].message.contains("compact requested"));
|
||||
|
||||
@@ -572,7 +572,7 @@ async fn run_e2e_rewind_fixture(
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
greeting: Greeting {
|
||||
worker_name: worker_name.clone(),
|
||||
cwd: workspace_root.display().to_string(),
|
||||
@@ -1438,13 +1438,15 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
|
||||
WorkerStatus::Running | WorkerStatus::Paused
|
||||
) {
|
||||
app.shutdown_confirm = None;
|
||||
return Some(Method::Cancel);
|
||||
let command = app.next_command_envelope();
|
||||
return Some(Method::Cancel { command });
|
||||
}
|
||||
if let Some(pressed_at) = app.shutdown_confirm
|
||||
&& pressed_at.elapsed() < CONFIRM_TIMEOUT
|
||||
{
|
||||
app.shutdown_confirm = None;
|
||||
return Some(Method::Shutdown);
|
||||
let command = app.next_command_envelope();
|
||||
return Some(Method::Shutdown { command });
|
||||
}
|
||||
app.shutdown_confirm = Some(std::time::Instant::now());
|
||||
app.flash_actionbar_notice(
|
||||
@@ -1460,7 +1462,8 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
|
||||
/// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running).
|
||||
fn handle_pause_or_quit(app: &mut App) -> Option<Method> {
|
||||
if app.worker_status == WorkerStatus::Running {
|
||||
return Some(Method::Pause);
|
||||
let command = app.next_command_envelope();
|
||||
return Some(Method::Pause { command });
|
||||
}
|
||||
if let Some(t) = app.quit_confirm
|
||||
&& t.elapsed() < CONFIRM_TIMEOUT
|
||||
@@ -2090,7 +2093,7 @@ mod tests {
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL),
|
||||
),
|
||||
Some(Method::Pause)
|
||||
Some(Method::Pause { .. })
|
||||
));
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
|
||||
@@ -2100,7 +2103,7 @@ mod tests {
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
||||
),
|
||||
Some(Method::Cancel)
|
||||
Some(Method::Cancel { .. })
|
||||
));
|
||||
assert_eq!(app.queued_input_count(), 1);
|
||||
}
|
||||
@@ -2114,7 +2117,7 @@ mod tests {
|
||||
&mut app,
|
||||
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
||||
);
|
||||
assert!(matches!(cancel, Some(Method::Cancel)));
|
||||
assert!(matches!(cancel, Some(Method::Cancel { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2136,7 +2139,7 @@ mod tests {
|
||||
|
||||
assert!(matches!(
|
||||
handle_key(&mut app, ctrl_x()),
|
||||
Some(Method::Shutdown)
|
||||
Some(Method::Shutdown { .. })
|
||||
));
|
||||
assert!(app.shutdown_confirm.is_none());
|
||||
}
|
||||
@@ -2466,7 +2469,7 @@ mod tests {
|
||||
}
|
||||
|
||||
let method = handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
||||
assert!(matches!(method, Some(protocol::Method::Compact)));
|
||||
assert!(matches!(method, Some(protocol::Method::Compact { .. })));
|
||||
assert!(!app.is_command_mode());
|
||||
assert_eq!(input_text(&app), "");
|
||||
assert_eq!(app.queued_input_count(), 0);
|
||||
@@ -2573,7 +2576,7 @@ mod tests {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
@@ -2606,7 +2609,7 @@ mod tests {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: vec![],
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
});
|
||||
@@ -2743,8 +2746,8 @@ mod tests {
|
||||
kind: protocol::InternalWorkerKind::SubWorker,
|
||||
},
|
||||
revision: 1,
|
||||
event: Box::new(Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
event: Box::new(Event::WorkerState {
|
||||
snapshot: WorkerStatus::Running.into(),
|
||||
}),
|
||||
});
|
||||
enter_command_mode(&mut app);
|
||||
@@ -2859,8 +2862,8 @@ mod tests {
|
||||
kind: protocol::InternalWorkerKind::SubWorker,
|
||||
},
|
||||
revision: 1,
|
||||
event: Box::new(Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
event: Box::new(Event::WorkerState {
|
||||
snapshot: WorkerStatus::Running.into(),
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -2885,8 +2888,8 @@ mod tests {
|
||||
kind: protocol::InternalWorkerKind::SubWorker,
|
||||
},
|
||||
revision: 1,
|
||||
event: Box::new(Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
event: Box::new(Event::WorkerState {
|
||||
snapshot: WorkerStatus::Running.into(),
|
||||
}),
|
||||
});
|
||||
handle_key(&mut app, key(KeyCode::Tab));
|
||||
@@ -2902,7 +2905,7 @@ mod tests {
|
||||
);
|
||||
|
||||
assert!(first.is_none());
|
||||
assert!(matches!(second, Some(Method::Shutdown)));
|
||||
assert!(matches!(second, Some(Method::Shutdown { .. })));
|
||||
assert_eq!(app.worker_status, WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
@@ -2924,8 +2927,8 @@ mod tests {
|
||||
kind: protocol::InternalWorkerKind::SubWorker,
|
||||
},
|
||||
revision: 1,
|
||||
event: Box::new(Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
event: Box::new(Event::WorkerState {
|
||||
snapshot: WorkerStatus::Running.into(),
|
||||
}),
|
||||
});
|
||||
|
||||
|
||||
@@ -307,6 +307,8 @@ pub struct WorkerSummary {
|
||||
pub worker_id: WorkerId,
|
||||
pub status: WorkerStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub workspace_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||
@@ -325,6 +327,8 @@ pub struct WorkerDetail {
|
||||
pub worker_id: WorkerId,
|
||||
pub status: WorkerStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub workspace_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||
@@ -341,6 +345,8 @@ pub struct WorkerDetail {
|
||||
pub struct WorkerLifecycleAck {
|
||||
pub worker_ref: WorkerRef,
|
||||
pub status: WorkerStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -15,18 +15,6 @@ use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use workdir::WorkdirSessionHandle;
|
||||
|
||||
/// Current execution-side run state for a Worker.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkerExecutionRunState {
|
||||
#[default]
|
||||
Stopped,
|
||||
Idle,
|
||||
Busy,
|
||||
Rejected,
|
||||
Errored,
|
||||
}
|
||||
|
||||
/// Execution operation that produced a result.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -55,7 +43,8 @@ pub struct WorkerSubmissionAck {
|
||||
pub struct WorkerExecutionResult {
|
||||
pub operation: WorkerExecutionOperation,
|
||||
pub outcome: WorkerExecutionOutcome,
|
||||
pub run_state: WorkerExecutionRunState,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub message: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -74,22 +63,23 @@ pub enum WorkerExecutionOutcome {
|
||||
}
|
||||
|
||||
impl WorkerExecutionResult {
|
||||
pub fn accepted(
|
||||
operation: WorkerExecutionOperation,
|
||||
run_state: WorkerExecutionRunState,
|
||||
) -> Self {
|
||||
pub fn accepted(operation: WorkerExecutionOperation) -> Self {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Accepted,
|
||||
run_state,
|
||||
worker_state: None,
|
||||
message: None,
|
||||
submission: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_worker_state(mut self, worker_state: protocol::WorkerStateSnapshot) -> Self {
|
||||
self.worker_state = Some(worker_state);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn accepted_submission(
|
||||
operation: WorkerExecutionOperation,
|
||||
run_state: WorkerExecutionRunState,
|
||||
submission_request_id: impl Into<String>,
|
||||
submission_id: impl Into<String>,
|
||||
disposition: protocol::SubmissionDisposition,
|
||||
@@ -97,7 +87,7 @@ impl WorkerExecutionResult {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Accepted,
|
||||
run_state,
|
||||
worker_state: None,
|
||||
message: None,
|
||||
submission: Some(WorkerSubmissionAck {
|
||||
submission_request_id: submission_request_id.into(),
|
||||
@@ -111,7 +101,7 @@ impl WorkerExecutionResult {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Busy,
|
||||
run_state: WorkerExecutionRunState::Busy,
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
}
|
||||
@@ -121,7 +111,7 @@ impl WorkerExecutionResult {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Rejected,
|
||||
run_state: WorkerExecutionRunState::Stopped,
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
}
|
||||
@@ -131,7 +121,7 @@ impl WorkerExecutionResult {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Errored,
|
||||
run_state: WorkerExecutionRunState::Errored,
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
}
|
||||
@@ -141,7 +131,7 @@ impl WorkerExecutionResult {
|
||||
Self {
|
||||
operation,
|
||||
outcome: WorkerExecutionOutcome::Unsupported,
|
||||
run_state: WorkerExecutionRunState::Stopped,
|
||||
worker_state: None,
|
||||
message: Some(message.into()),
|
||||
submission: None,
|
||||
}
|
||||
@@ -280,7 +270,6 @@ pub struct WorkerExecutionRestoreRequest {
|
||||
pub enum WorkerExecutionSpawnResult {
|
||||
Connected {
|
||||
handle: WorkerExecutionHandle,
|
||||
run_state: WorkerExecutionRunState,
|
||||
working_directory: Option<WorkingDirectoryStatus>,
|
||||
},
|
||||
Rejected(WorkerExecutionResult),
|
||||
@@ -290,12 +279,10 @@ pub enum WorkerExecutionSpawnResult {
|
||||
impl WorkerExecutionSpawnResult {
|
||||
pub fn connected(
|
||||
handle: WorkerExecutionHandle,
|
||||
run_state: WorkerExecutionRunState,
|
||||
working_directory: Option<WorkingDirectoryStatus>,
|
||||
) -> Self {
|
||||
Self::Connected {
|
||||
handle,
|
||||
run_state,
|
||||
working_directory,
|
||||
}
|
||||
}
|
||||
@@ -623,7 +610,6 @@ mod tests {
|
||||
fn submission_ack_survives_json_round_trip() {
|
||||
let result = WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
"request-1",
|
||||
"submission-1",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
|
||||
@@ -2205,8 +2205,8 @@ mod tests {
|
||||
};
|
||||
use crate::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
|
||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest,
|
||||
WorkerExecutionSpawnResult,
|
||||
};
|
||||
use crate::management::RuntimeOptions;
|
||||
use axum::body::to_bytes;
|
||||
@@ -2895,7 +2895,6 @@ mod tests {
|
||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request
|
||||
.working_directory
|
||||
.as_ref()
|
||||
@@ -2909,7 +2908,6 @@ mod tests {
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request.previous_working_directory,
|
||||
}
|
||||
}
|
||||
@@ -2922,24 +2920,17 @@ mod tests {
|
||||
if let Some(submission_id) = input.submission_request_id {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
submission_id.clone(),
|
||||
submission_id,
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Stop,
|
||||
WorkerExecutionRunState::Stopped,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3211,8 +3202,7 @@ mod ws_tests {
|
||||
};
|
||||
use crate::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||
WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest,
|
||||
WorkerExecutionSpawnResult,
|
||||
WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
};
|
||||
use crate::management::RuntimeOptions;
|
||||
use futures::{SinkExt, StreamExt};
|
||||
@@ -3232,7 +3222,6 @@ mod ws_tests {
|
||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request
|
||||
.working_directory
|
||||
.as_ref()
|
||||
@@ -3248,28 +3237,46 @@ mod ws_tests {
|
||||
if let Some(submission_id) = input.submission_request_id {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
submission_id.clone(),
|
||||
submission_id,
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option<protocol::Event> {
|
||||
Some(protocol::Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
worker_name: handle.worker_ref().worker_id.to_string(),
|
||||
cwd: String::new(),
|
||||
provider: "ws-test".to_string(),
|
||||
model: "ws-test".to_string(),
|
||||
scope_summary: "WebSocket test execution snapshot".to_string(),
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
state: protocol::WorkerStateSnapshot::initial(1),
|
||||
in_flight: protocol::InFlightSnapshot {
|
||||
blocks: Vec::new(),
|
||||
commands: Vec::new(),
|
||||
},
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn dispatch_method(
|
||||
&self,
|
||||
_handle: &WorkerExecutionHandle,
|
||||
_method: protocol::Method,
|
||||
) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::ProtocolMethod,
|
||||
WorkerExecutionRunState::Idle,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::ProtocolMethod)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3480,16 +3487,16 @@ mod ws_tests {
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
&other.worker_ref,
|
||||
protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Running,
|
||||
protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStatus::Running.into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
&worker_ref,
|
||||
protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Running,
|
||||
protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStatus::Running.into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3504,7 +3511,16 @@ mod ws_tests {
|
||||
..
|
||||
}) if delivered_subscription_id == subscription_id
|
||||
&& worker.worker_id.as_str() == worker_ref.worker_id.to_string()
|
||||
&& worker.state == protocol::subscription::SubscriptionWorkerState::Running
|
||||
&& worker.state == protocol::subscription::SubscriptionWorkerState::Idle
|
||||
&& matches!(
|
||||
worker.worker_state,
|
||||
Some(protocol::WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running
|
||||
)),
|
||||
..
|
||||
})
|
||||
)
|
||||
));
|
||||
|
||||
let unsubscribe_request_id =
|
||||
|
||||
@@ -13,8 +13,8 @@ use crate::error::RuntimeError;
|
||||
use crate::execution::WorkerExecutionRestoreRequest;
|
||||
use crate::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle,
|
||||
WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionRunState,
|
||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest,
|
||||
WorkerExecutionSpawnResult,
|
||||
};
|
||||
#[cfg(feature = "fs-store")]
|
||||
use crate::fs_store::{
|
||||
@@ -700,6 +700,7 @@ impl Runtime {
|
||||
worker_ref: worker_ref.clone(),
|
||||
worker_id: worker_id.clone(),
|
||||
status: WorkerStatus::Stopped,
|
||||
worker_state: None,
|
||||
workspace_id: scope.map(|scope| scope.workspace_id.clone()),
|
||||
request: durable_request,
|
||||
run_generation: 1,
|
||||
@@ -725,12 +726,11 @@ impl Runtime {
|
||||
};
|
||||
|
||||
let spawn_result = backend.spawn_worker(spawn_request);
|
||||
let (handle, run_state, working_directory) = match spawn_result {
|
||||
let (handle, working_directory) = match spawn_result {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle,
|
||||
run_state,
|
||||
working_directory,
|
||||
} => (handle, run_state, working_directory),
|
||||
} => (handle, working_directory),
|
||||
WorkerExecutionSpawnResult::Rejected(result)
|
||||
| WorkerExecutionSpawnResult::Errored(result) => {
|
||||
self.rollback_failed_create(&worker_ref)?;
|
||||
@@ -785,11 +785,9 @@ impl Runtime {
|
||||
result,
|
||||
});
|
||||
}
|
||||
let initial_run_state = dispatch_result.run_state;
|
||||
let detail = self.commit_created_worker(
|
||||
&worker_ref,
|
||||
handle,
|
||||
initial_run_state,
|
||||
working_directory,
|
||||
dispatch_result,
|
||||
)?;
|
||||
@@ -799,9 +797,8 @@ impl Runtime {
|
||||
self.commit_created_worker(
|
||||
&worker_ref,
|
||||
handle,
|
||||
run_state,
|
||||
working_directory,
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn, run_state),
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1086,13 +1083,12 @@ impl Runtime {
|
||||
match backend.restore_worker(request) {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle,
|
||||
run_state,
|
||||
working_directory,
|
||||
} => {
|
||||
self.commit_restored_worker_execution(
|
||||
worker_ref,
|
||||
handle,
|
||||
run_state,
|
||||
WorkerStatus::Idle,
|
||||
working_directory,
|
||||
)?;
|
||||
self.worker_detail(worker_ref)
|
||||
@@ -1222,7 +1218,9 @@ impl Runtime {
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
worker.status = worker_status_from_run_state(dispatch_result.run_state);
|
||||
if let Some(snapshot) = dispatch_result.worker_state.as_ref() {
|
||||
let _ = worker.apply_worker_state(snapshot);
|
||||
}
|
||||
let status = worker.status;
|
||||
#[cfg(feature = "ws-server")]
|
||||
if let Some(payload) = input_protocol_event(&input) {
|
||||
@@ -1431,7 +1429,7 @@ impl Runtime {
|
||||
let entries = self.worker_completions(worker_ref, kind, &prefix)?;
|
||||
return Ok(vec![Event::Completions { kind, entries }]);
|
||||
}
|
||||
if matches!(&method, Method::Shutdown) {
|
||||
if matches!(&method, Method::Shutdown { .. }) {
|
||||
self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?;
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
@@ -1481,16 +1479,19 @@ impl Runtime {
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
handle: WorkerExecutionHandle,
|
||||
run_state: WorkerExecutionRunState,
|
||||
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
||||
_result: WorkerExecutionResult,
|
||||
result: WorkerExecutionResult,
|
||||
) -> Result<WorkerDetail, RuntimeError> {
|
||||
let mut state = self.lock()?;
|
||||
let detail = {
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
worker.execution_handle = Some(handle);
|
||||
worker.execution_bound = true;
|
||||
worker.status = worker_status_from_run_state(run_state);
|
||||
worker.status = WorkerStatus::Idle;
|
||||
worker.worker_state = None;
|
||||
if let Some(snapshot) = result.worker_state.as_ref() {
|
||||
let _ = worker.apply_worker_state(snapshot);
|
||||
}
|
||||
worker.restore_intent = restore_intent_for_status(worker.status);
|
||||
worker.working_directory = working_directory;
|
||||
worker.detail()
|
||||
@@ -1518,16 +1519,26 @@ impl Runtime {
|
||||
worker_ref: &WorkerRef,
|
||||
result: WorkerExecutionResult,
|
||||
) -> Result<(), RuntimeError> {
|
||||
// Accepted dispatch without a state snapshot is transport evidence only;
|
||||
// the revisioned protocol stream remains live authority. Test/detached
|
||||
// backends may return an exact full snapshot as their acknowledgement.
|
||||
if !result.is_accepted() {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(snapshot) = result.worker_state else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut state = self.lock()?;
|
||||
if result.is_accepted() {
|
||||
let status = worker_status_from_run_state(result.run_state);
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
worker.status = status;
|
||||
worker.restore_intent = restore_intent_for_status(status);
|
||||
let applied = worker
|
||||
.apply_worker_state(&snapshot)
|
||||
.is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied));
|
||||
if !applied {
|
||||
return Ok(());
|
||||
}
|
||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||
state.persist_runtime_snapshot()?;
|
||||
state.persist_worker(&worker_ref.worker_id)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1615,20 +1626,26 @@ impl Runtime {
|
||||
worker_ref: &WorkerRef,
|
||||
reason: Option<String>,
|
||||
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
||||
let current = {
|
||||
{
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
state.worker(worker_ref)?.status
|
||||
};
|
||||
if matches!(current, WorkerStatus::Idle | WorkerStatus::Stopped) {
|
||||
if state.worker(worker_ref)?.status == WorkerStatus::Stopped {
|
||||
return Ok(WorkerLifecycleAck {
|
||||
worker_ref: worker_ref.clone(),
|
||||
status: current,
|
||||
status: WorkerStatus::Stopped,
|
||||
worker_state: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?;
|
||||
let _ = reason;
|
||||
self.transition_worker_preserving_execution(worker_ref, WorkerStatus::Idle)
|
||||
let state = self.lock()?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
Ok(WorkerLifecycleAck {
|
||||
worker_ref: worker_ref.clone(),
|
||||
status: worker.status,
|
||||
worker_state: worker.worker_state.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Delete a non-running Worker through a workspace-scoped Runtime authorization context.
|
||||
@@ -1715,27 +1732,9 @@ impl Runtime {
|
||||
return Ok(snapshot);
|
||||
}
|
||||
}
|
||||
Ok(protocol::Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
worker_name: worker_ref.worker_id.to_string(),
|
||||
cwd: String::new(),
|
||||
provider: "worker-runtime".to_string(),
|
||||
model: "worker-runtime".to_string(),
|
||||
scope_summary: "runtime worker observation".to_string(),
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: protocol::WorkerStatus::Idle,
|
||||
in_flight: protocol::InFlightSnapshot {
|
||||
blocks: Vec::new(),
|
||||
commands: Vec::new(),
|
||||
},
|
||||
internal_workers: Vec::new(),
|
||||
Err(RuntimeError::WorkerExecutionUnavailable {
|
||||
worker_id: worker_ref.worker_id,
|
||||
message: "authoritative Worker snapshot is unavailable".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1774,12 +1773,13 @@ impl Runtime {
|
||||
) -> Result<WorkerObservationEvent, RuntimeError> {
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_worker_ref(worker_ref)?;
|
||||
let status_changed = state.project_protocol_event_to_status(worker_ref, &payload);
|
||||
let worker_state_changed =
|
||||
state.project_protocol_event_to_worker_state(worker_ref, &payload);
|
||||
let activity_changed = state.project_internal_worker_activity(worker_ref, &payload);
|
||||
if status_changed || activity_changed {
|
||||
if worker_state_changed || activity_changed {
|
||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||
}
|
||||
if status_changed {
|
||||
if worker_state_changed {
|
||||
state.persist_runtime_snapshot()?;
|
||||
state.persist_worker(&worker_ref.worker_id)?;
|
||||
}
|
||||
@@ -1815,26 +1815,6 @@ impl Runtime {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn transition_worker_preserving_execution(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
status: WorkerStatus,
|
||||
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
worker.status = status;
|
||||
worker.restore_intent = restore_intent_for_status(status);
|
||||
let status = worker.status;
|
||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||
state.persist_runtime_snapshot()?;
|
||||
state.persist_worker(&worker_ref.worker_id)?;
|
||||
Ok(WorkerLifecycleAck {
|
||||
worker_ref: worker_ref.clone(),
|
||||
status,
|
||||
})
|
||||
}
|
||||
|
||||
fn transition_worker(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
@@ -1846,6 +1826,7 @@ impl Runtime {
|
||||
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
worker.status = status;
|
||||
worker.worker_state = None;
|
||||
worker.restore_intent = restore_intent_for_status(status);
|
||||
worker.execution_handle = None;
|
||||
worker.internal_workers.clear();
|
||||
@@ -1856,6 +1837,7 @@ impl Runtime {
|
||||
Ok(WorkerLifecycleAck {
|
||||
worker_ref: worker_ref.clone(),
|
||||
status,
|
||||
worker_state: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1968,12 +1950,11 @@ impl Runtime {
|
||||
match backend.restore_worker(request) {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle,
|
||||
run_state,
|
||||
working_directory,
|
||||
} => self.commit_restored_worker_execution(
|
||||
&candidate.worker_ref,
|
||||
handle,
|
||||
run_state,
|
||||
WorkerStatus::Idle,
|
||||
working_directory,
|
||||
)?,
|
||||
WorkerExecutionSpawnResult::Rejected(result)
|
||||
@@ -1990,7 +1971,7 @@ impl Runtime {
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
handle: WorkerExecutionHandle,
|
||||
run_state: WorkerExecutionRunState,
|
||||
status: WorkerStatus,
|
||||
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
||||
) -> Result<(), RuntimeError> {
|
||||
let mut state = self.lock()?;
|
||||
@@ -1999,7 +1980,7 @@ impl Runtime {
|
||||
let worker = state.worker_mut(worker_ref)?;
|
||||
worker.execution_handle = Some(handle);
|
||||
worker.execution_bound = true;
|
||||
worker.status = worker_status_from_run_state(run_state);
|
||||
worker.status = status;
|
||||
worker.restore_intent = restore_intent_for_status(worker.status);
|
||||
worker.working_directory = working_directory;
|
||||
}
|
||||
@@ -2281,6 +2262,7 @@ impl RuntimeState {
|
||||
worker_ref: worker.worker_ref,
|
||||
worker_id: worker.worker_id,
|
||||
status: worker.status,
|
||||
worker_state: None,
|
||||
workspace_id: worker.workspace_id,
|
||||
request: worker.request,
|
||||
run_generation,
|
||||
@@ -2610,6 +2592,7 @@ impl RuntimeState {
|
||||
.get(&worker.worker_id)
|
||||
.copied()
|
||||
.unwrap_or(0),
|
||||
worker_state: worker.worker_state.clone(),
|
||||
state: subscription_worker_state(worker.status),
|
||||
has_running_internal_workers: worker
|
||||
.internal_workers
|
||||
@@ -2867,7 +2850,7 @@ impl RuntimeState {
|
||||
) {
|
||||
match event {
|
||||
protocol::Event::Snapshot {
|
||||
status,
|
||||
state,
|
||||
internal_workers,
|
||||
..
|
||||
} => {
|
||||
@@ -2875,7 +2858,7 @@ impl RuntimeState {
|
||||
statuses.insert(
|
||||
worker.session_id.clone(),
|
||||
InternalWorkerActivity {
|
||||
status: *status,
|
||||
status: state.catalog_status(),
|
||||
parent_session_id: worker.parent_session_id.clone(),
|
||||
},
|
||||
);
|
||||
@@ -2888,26 +2871,17 @@ impl RuntimeState {
|
||||
event,
|
||||
..
|
||||
} => Self::project_internal_worker_event(statuses, nested_worker, event),
|
||||
protocol::Event::Status { status } => {
|
||||
statuses.insert(
|
||||
worker.session_id.clone(),
|
||||
InternalWorkerActivity {
|
||||
status: *status,
|
||||
parent_session_id: worker.parent_session_id.clone(),
|
||||
protocol::Event::WorkerState { snapshot }
|
||||
| protocol::Event::CommandAcknowledged {
|
||||
acknowledgement:
|
||||
protocol::WorkerCommandAcknowledgement {
|
||||
state: snapshot, ..
|
||||
},
|
||||
);
|
||||
}
|
||||
protocol::Event::RunEnd { result } => {
|
||||
let status = match result {
|
||||
protocol::RunResult::Paused => protocol::WorkerStatus::Paused,
|
||||
protocol::RunResult::Finished
|
||||
| protocol::RunResult::LimitReached
|
||||
| protocol::RunResult::RolledBack => protocol::WorkerStatus::Idle,
|
||||
};
|
||||
} => {
|
||||
statuses.insert(
|
||||
worker.session_id.clone(),
|
||||
InternalWorkerActivity {
|
||||
status,
|
||||
status: snapshot.catalog_status(),
|
||||
parent_session_id: worker.parent_session_id.clone(),
|
||||
},
|
||||
);
|
||||
@@ -2954,7 +2928,7 @@ impl RuntimeState {
|
||||
Self::update_internal_worker_activity(&mut worker.internal_workers, event)
|
||||
}
|
||||
|
||||
fn project_protocol_event_to_status(
|
||||
fn project_protocol_event_to_worker_state(
|
||||
&mut self,
|
||||
worker_ref: &WorkerRef,
|
||||
event: &protocol::Event,
|
||||
@@ -2962,38 +2936,26 @@ impl RuntimeState {
|
||||
let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else {
|
||||
return false;
|
||||
};
|
||||
let next_status = match event {
|
||||
protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Running,
|
||||
} => Some(WorkerStatus::Running),
|
||||
protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Idle,
|
||||
} => Some(WorkerStatus::Idle),
|
||||
protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Paused,
|
||||
} => Some(WorkerStatus::Paused),
|
||||
protocol::Event::Snapshot { status, .. } => match status {
|
||||
protocol::WorkerStatus::Running => Some(WorkerStatus::Running),
|
||||
protocol::WorkerStatus::Idle => Some(WorkerStatus::Idle),
|
||||
protocol::WorkerStatus::Paused => Some(WorkerStatus::Paused),
|
||||
protocol::WorkerStatus::Stopped => Some(WorkerStatus::Stopped),
|
||||
},
|
||||
protocol::Event::RunEnd { result } => match result {
|
||||
protocol::RunResult::Finished | protocol::RunResult::RolledBack => {
|
||||
Some(WorkerStatus::Idle)
|
||||
let incoming = match event {
|
||||
protocol::Event::WorkerState { snapshot }
|
||||
| protocol::Event::Snapshot {
|
||||
state: snapshot, ..
|
||||
}
|
||||
protocol::RunResult::Paused => Some(WorkerStatus::Paused),
|
||||
protocol::RunResult::LimitReached => Some(WorkerStatus::Idle),
|
||||
| protocol::Event::CommandAcknowledged {
|
||||
acknowledgement:
|
||||
protocol::WorkerCommandAcknowledgement {
|
||||
state: snapshot, ..
|
||||
},
|
||||
_ => None,
|
||||
} => snapshot,
|
||||
_ => return false,
|
||||
};
|
||||
if let Some(next_status) = next_status {
|
||||
let changed = worker.status != next_status;
|
||||
worker.status = next_status;
|
||||
worker.restore_intent = restore_intent_for_status(next_status);
|
||||
changed
|
||||
} else {
|
||||
false
|
||||
match worker.apply_worker_state(incoming) {
|
||||
Ok(protocol::WorkerStateSnapshotApply::Applied) => true,
|
||||
Ok(
|
||||
protocol::WorkerStateSnapshotApply::Duplicate
|
||||
| protocol::WorkerStateSnapshotApply::Stale,
|
||||
)
|
||||
| Err(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3009,6 +2971,7 @@ struct WorkerRecord {
|
||||
worker_ref: WorkerRef,
|
||||
worker_id: WorkerId,
|
||||
status: WorkerStatus,
|
||||
worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||
workspace_id: Option<String>,
|
||||
request: CreateWorkerRequest,
|
||||
run_generation: u64,
|
||||
@@ -3020,6 +2983,19 @@ struct WorkerRecord {
|
||||
}
|
||||
|
||||
impl WorkerRecord {
|
||||
fn apply_worker_state(
|
||||
&mut self,
|
||||
incoming: &protocol::WorkerStateSnapshot,
|
||||
) -> Result<protocol::WorkerStateSnapshotApply, protocol::WorkerStateSnapshotConflict> {
|
||||
match self.worker_state.as_mut() {
|
||||
Some(current) => protocol::apply_worker_state_snapshot(current, incoming),
|
||||
None => {
|
||||
self.worker_state = Some(incoming.clone());
|
||||
Ok(protocol::WorkerStateSnapshotApply::Applied)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
|
||||
self.workspace_id.as_deref() == Some(workspace_id)
|
||||
}
|
||||
@@ -3029,6 +3005,7 @@ impl WorkerRecord {
|
||||
worker_ref: self.worker_ref.clone(),
|
||||
worker_id: self.worker_id,
|
||||
status: self.status,
|
||||
worker_state: self.worker_state.clone(),
|
||||
workspace_id: self.workspace_id.clone(),
|
||||
working_directory: self.working_directory.clone(),
|
||||
profile: self.request.profile.clone(),
|
||||
@@ -3043,6 +3020,7 @@ impl WorkerRecord {
|
||||
worker_ref: self.worker_ref.clone(),
|
||||
worker_id: self.worker_id,
|
||||
status: self.status,
|
||||
worker_state: self.worker_state.clone(),
|
||||
workspace_id: self.workspace_id.clone(),
|
||||
working_directory: self.working_directory.clone(),
|
||||
profile: self.request.profile.clone(),
|
||||
@@ -3081,16 +3059,6 @@ fn restore_intent_for_status(status: WorkerStatus) -> WorkerRestoreIntent {
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_status_from_run_state(run_state: WorkerExecutionRunState) -> WorkerStatus {
|
||||
match run_state {
|
||||
WorkerExecutionRunState::Idle => WorkerStatus::Idle,
|
||||
WorkerExecutionRunState::Busy => WorkerStatus::Running,
|
||||
WorkerExecutionRunState::Stopped
|
||||
| WorkerExecutionRunState::Rejected
|
||||
| WorkerExecutionRunState::Errored => WorkerStatus::Stopped,
|
||||
}
|
||||
}
|
||||
|
||||
fn repository_resource_error(error: BackendResourceError) -> RuntimeError {
|
||||
let (code, message) = match error {
|
||||
BackendResourceError::Expired => (
|
||||
@@ -3304,7 +3272,7 @@ mod tests {
|
||||
};
|
||||
use crate::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle,
|
||||
WorkerExecutionRestoreRequest, WorkerExecutionRunState,
|
||||
WorkerExecutionRestoreRequest,
|
||||
};
|
||||
use crate::working_directory::WorkingDirectoryDiagnostic;
|
||||
use async_trait::async_trait;
|
||||
@@ -3313,6 +3281,14 @@ mod tests {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
fn test_command() -> protocol::WorkerCommandEnvelope {
|
||||
protocol::WorkerCommandEnvelope {
|
||||
command_id: 1,
|
||||
expected_execution_generation: 1,
|
||||
expected_worker_state_revision: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_resource_failures_keep_typed_credential_diagnostics() {
|
||||
let cases = [
|
||||
@@ -3359,7 +3335,9 @@ mod tests {
|
||||
protocol::Event::InternalWorker {
|
||||
worker,
|
||||
revision: 1,
|
||||
event: Box::new(protocol::Event::Status { status }),
|
||||
event: Box::new(protocol::Event::WorkerState {
|
||||
snapshot: status.into(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3452,7 +3430,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: protocol::WorkerStatus::Idle,
|
||||
state: protocol::WorkerStatus::Idle.into(),
|
||||
in_flight: protocol::InFlightSnapshot::default(),
|
||||
internal_workers: Vec::new(),
|
||||
};
|
||||
@@ -3967,7 +3945,6 @@ mod tests {
|
||||
.insert(request.worker_ref.worker_id.clone(), request.context);
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request
|
||||
.working_directory
|
||||
.as_ref()
|
||||
@@ -3997,7 +3974,6 @@ mod tests {
|
||||
.insert(request.worker_ref.worker_id.clone(), request.context);
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request
|
||||
.working_directory
|
||||
.as_ref()
|
||||
@@ -4020,7 +3996,6 @@ mod tests {
|
||||
.unwrap_or_else(|| {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
"request-test",
|
||||
"test-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
@@ -4038,17 +4013,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Stop,
|
||||
WorkerExecutionRunState::Stopped,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
|
||||
}
|
||||
|
||||
fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Cancel,
|
||||
WorkerExecutionRunState::Stopped,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Cancel)
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
@@ -4374,7 +4343,9 @@ mod tests {
|
||||
.send_protocol_method_scoped(
|
||||
&scope("workspace-a", "server-a"),
|
||||
&workspace_b.worker_ref,
|
||||
Method::Shutdown,
|
||||
Method::Shutdown {
|
||||
command: test_command(),
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
@@ -4722,11 +4693,10 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_worker_uses_committed_input_ack_run_state() {
|
||||
fn create_worker_does_not_infer_state_from_started_submission_ack() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
"request-test",
|
||||
"test-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
@@ -4737,6 +4707,68 @@ mod tests {
|
||||
let detail = runtime.create_worker(request).unwrap();
|
||||
|
||||
assert_eq!(detail.status, WorkerStatus::Idle);
|
||||
assert_eq!(detail.worker_state, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_applies_only_newer_worker_state_snapshots() {
|
||||
let (runtime, _) = runtime_and_backend();
|
||||
let detail = runtime
|
||||
.create_worker(task_request("state ordering"))
|
||||
.unwrap();
|
||||
let running = protocol::WorkerStateSnapshot {
|
||||
execution_generation: 7,
|
||||
revision: 3,
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)),
|
||||
last_command_id: 2,
|
||||
};
|
||||
assert!({
|
||||
let mut state = runtime.lock().unwrap();
|
||||
state.project_protocol_event_to_worker_state(
|
||||
&detail.worker_ref,
|
||||
&protocol::Event::WorkerState {
|
||||
snapshot: running.clone(),
|
||||
},
|
||||
)
|
||||
});
|
||||
assert_eq!(
|
||||
runtime
|
||||
.worker_detail(&detail.worker_ref)
|
||||
.unwrap()
|
||||
.worker_state,
|
||||
Some(running.clone())
|
||||
);
|
||||
|
||||
assert!({
|
||||
let mut state = runtime.lock().unwrap();
|
||||
!state.project_protocol_event_to_worker_state(
|
||||
&detail.worker_ref,
|
||||
&protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStateSnapshot {
|
||||
revision: 2,
|
||||
state: protocol::WorkerState::Idle,
|
||||
..running.clone()
|
||||
},
|
||||
},
|
||||
)
|
||||
});
|
||||
assert!({
|
||||
let mut state = runtime.lock().unwrap();
|
||||
!state.project_protocol_event_to_worker_state(
|
||||
&detail.worker_ref,
|
||||
&protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Idle,
|
||||
..running.clone()
|
||||
},
|
||||
},
|
||||
)
|
||||
});
|
||||
let after = runtime.worker_detail(&detail.worker_ref).unwrap();
|
||||
assert_eq!(after.status, WorkerStatus::Idle);
|
||||
assert_eq!(after.worker_state, Some(running));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4745,7 +4777,6 @@ mod tests {
|
||||
backend.preserve_commit_ack_submission_id();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
"request-test",
|
||||
"forged-submission",
|
||||
protocol::SubmissionDisposition::Started,
|
||||
@@ -4770,7 +4801,6 @@ mod tests {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
backend.set_dispatch_result(WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
));
|
||||
let mut request = task_request("missing initial input commit ack");
|
||||
request.initial_input = Some(WorkerInput::user("start the ticket"));
|
||||
@@ -4898,7 +4928,7 @@ mod tests {
|
||||
context_window: 128,
|
||||
context_tokens: 64,
|
||||
},
|
||||
status: protocol::WorkerStatus::Running,
|
||||
state: protocol::WorkerStatus::Running.into(),
|
||||
in_flight: protocol::InFlightSnapshot {
|
||||
blocks: Vec::new(),
|
||||
commands: Vec::new(),
|
||||
@@ -4914,18 +4944,38 @@ mod tests {
|
||||
protocol::Event::Snapshot {
|
||||
session,
|
||||
greeting,
|
||||
status,
|
||||
state,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(session.entries.len(), 1);
|
||||
assert_eq!(session.entries[0].entry_id, "restored-log-entry");
|
||||
assert_eq!(greeting.worker_name, "live-worker");
|
||||
assert_eq!(status, protocol::WorkerStatus::Running);
|
||||
assert_eq!(state.catalog_status(), protocol::WorkerStatus::Running);
|
||||
}
|
||||
other => panic!("expected snapshot, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
#[test]
|
||||
fn observation_snapshot_fails_closed_when_backend_snapshot_is_unavailable() {
|
||||
let runtime = runtime_with_backend();
|
||||
let detail = runtime
|
||||
.create_worker(task_request("snapshot unavailable"))
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
runtime
|
||||
.worker_observation_snapshot(&detail.worker_ref)
|
||||
.unwrap_err(),
|
||||
RuntimeError::WorkerExecutionUnavailable {
|
||||
worker_id,
|
||||
message,
|
||||
} if worker_id == detail.worker_ref.worker_id
|
||||
&& message == "authoritative Worker snapshot is unavailable"
|
||||
));
|
||||
}
|
||||
|
||||
struct InputOnlyBackend;
|
||||
|
||||
impl WorkerExecutionBackend for InputOnlyBackend {
|
||||
@@ -4936,7 +4986,6 @@ mod tests {
|
||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request
|
||||
.working_directory
|
||||
.as_ref()
|
||||
@@ -4951,7 +5000,6 @@ mod tests {
|
||||
) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Idle,
|
||||
"request-test",
|
||||
input
|
||||
.submission_request_id
|
||||
@@ -4993,7 +5041,12 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Shutdown)
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Shutdown {
|
||||
command: test_command(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
@@ -5009,7 +5062,12 @@ mod tests {
|
||||
.create_worker(task_request("restore explicitly"))
|
||||
.unwrap();
|
||||
runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Shutdown)
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Shutdown {
|
||||
command: test_command(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
@@ -5025,10 +5083,9 @@ mod tests {
|
||||
|
||||
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
|
||||
assert_eq!(
|
||||
runtime.worker_detail(&detail.worker_ref).unwrap().status,
|
||||
WorkerStatus::Idle
|
||||
);
|
||||
let restored = runtime.worker_detail(&detail.worker_ref).unwrap();
|
||||
assert_eq!(restored.status, WorkerStatus::Idle);
|
||||
assert_eq!(restored.worker_state, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex, mpsc};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, RwLock, mpsc};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::auth::{
|
||||
@@ -25,8 +25,8 @@ use crate::catalog::{
|
||||
};
|
||||
use crate::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
|
||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest,
|
||||
WorkerExecutionSpawnResult,
|
||||
};
|
||||
use crate::identity::WorkerRef;
|
||||
use crate::interaction::{WorkerInput, WorkerInputKind};
|
||||
@@ -38,7 +38,28 @@ use crate::working_directory::{
|
||||
WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus};
|
||||
#[cfg(test)]
|
||||
use protocol::WorkerStatus;
|
||||
use protocol::{Event, Method, Segment, WorkerCommandEnvelope};
|
||||
|
||||
static NEXT_INTERNAL_COMMAND_ID: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
fn next_internal_command(
|
||||
state: &RwLock<protocol::WorkerStateSnapshot>,
|
||||
) -> Result<WorkerCommandEnvelope, String> {
|
||||
let snapshot = state
|
||||
.read()
|
||||
.map_err(|_| "worker state lock is poisoned".to_string())?
|
||||
.clone();
|
||||
let floor = snapshot.last_command_id.saturating_add(1);
|
||||
let command_id = NEXT_INTERNAL_COMMAND_ID
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
|
||||
Some(current.max(floor).saturating_add(1))
|
||||
})
|
||||
.unwrap_or(floor)
|
||||
.max(floor);
|
||||
Ok(WorkerCommandEnvelope::for_snapshot(command_id, &snapshot))
|
||||
}
|
||||
use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
|
||||
#[cfg(test)]
|
||||
use session_store::{FsStore, FsWorkerStore};
|
||||
@@ -172,7 +193,7 @@ impl WorkerObservationProvider for RuntimeGrantedWorkerObservationProvider {
|
||||
},
|
||||
display_name: grant.worker_id.clone(),
|
||||
relation: "granted_peer".to_string(),
|
||||
status: format!("{:?}", state.get_status()).to_lowercase(),
|
||||
status: format!("{:?}", state.catalog_status()).to_lowercase(),
|
||||
});
|
||||
}
|
||||
subjects.sort_by(|left, right| left.subject.cmp(&right.subject));
|
||||
@@ -1174,10 +1195,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RuntimeWorkerExecution {
|
||||
handle: WorkerHandle,
|
||||
shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>,
|
||||
busy: Arc<AtomicBool>,
|
||||
worker_state: Arc<RwLock<protocol::WorkerStateSnapshot>>,
|
||||
workspace_client: Option<Arc<dyn WorkspaceClient>>,
|
||||
}
|
||||
|
||||
@@ -1275,7 +1297,7 @@ where
|
||||
) -> Result<
|
||||
(
|
||||
WorkerHandle,
|
||||
Arc<AtomicBool>,
|
||||
Arc<RwLock<protocol::WorkerStateSnapshot>>,
|
||||
Option<Arc<dyn WorkspaceClient>>,
|
||||
),
|
||||
WorkerExecutionResult,
|
||||
@@ -1301,7 +1323,7 @@ where
|
||||
.map(|execution| {
|
||||
(
|
||||
execution.handle.clone(),
|
||||
execution.busy.clone(),
|
||||
execution.worker_state.clone(),
|
||||
execution.workspace_client.clone(),
|
||||
)
|
||||
})
|
||||
@@ -1318,7 +1340,6 @@ where
|
||||
operation: WorkerExecutionOperation,
|
||||
worker: WorkerHandle,
|
||||
method: Method,
|
||||
accepted_run_state: WorkerExecutionRunState,
|
||||
) -> WorkerExecutionResult {
|
||||
self.run_on_adapter_runtime(async move {
|
||||
worker
|
||||
@@ -1326,7 +1347,7 @@ where
|
||||
.await
|
||||
.map_err(|err| format!("failed to send Worker method: {err}"))
|
||||
})
|
||||
.map(|_| WorkerExecutionResult::accepted(operation, accepted_run_state))
|
||||
.map(|_| WorkerExecutionResult::accepted(operation))
|
||||
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
|
||||
}
|
||||
|
||||
@@ -1336,7 +1357,6 @@ where
|
||||
worker: WorkerHandle,
|
||||
method: Method,
|
||||
submission_request_id: String,
|
||||
accepted_run_state: WorkerExecutionRunState,
|
||||
) -> WorkerExecutionResult {
|
||||
let request_id = submission_request_id.clone();
|
||||
self.run_on_adapter_runtime(async move {
|
||||
@@ -1395,7 +1415,6 @@ where
|
||||
.map(|(submission_id, disposition)| {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
operation,
|
||||
accepted_run_state,
|
||||
submission_request_id,
|
||||
submission_id,
|
||||
disposition,
|
||||
@@ -1414,41 +1433,31 @@ where
|
||||
working_directory: Option<WorkingDirectoryBinding>,
|
||||
workspace_client: Option<Arc<dyn WorkspaceClient>>,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
let busy = Arc::new(AtomicBool::new(false));
|
||||
let worker_state = Arc::new(RwLock::new(handle.shared_state.snapshot()));
|
||||
#[cfg(feature = "ws-server")]
|
||||
{
|
||||
let streams = subscribe_worker_protocol_session(&handle);
|
||||
let mut events = streams.events;
|
||||
let mut entry_events = streams.log_entries;
|
||||
let bridge_busy = busy.clone();
|
||||
let bridge_worker_state = worker_state.clone();
|
||||
if let Err(message) = self.spawn_on_adapter_runtime(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = events.recv() => {
|
||||
match event {
|
||||
Ok(event) => {
|
||||
let next_busy = match &event {
|
||||
Event::InvokeStart { .. }
|
||||
| Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
} => Some(true),
|
||||
Event::RunEnd { .. }
|
||||
| Event::Error {
|
||||
code: ErrorCode::NotPaused,
|
||||
..
|
||||
}
|
||||
| Event::Status {
|
||||
status:
|
||||
WorkerStatus::Idle
|
||||
| WorkerStatus::Paused
|
||||
| WorkerStatus::Stopped,
|
||||
}
|
||||
| Event::Shutdown => Some(false),
|
||||
_ => None,
|
||||
};
|
||||
Ok(mut event) => {
|
||||
match apply_protocol_worker_state(&bridge_worker_state, &mut event) {
|
||||
Ok(true) => {
|
||||
let _ = bridge_context.publish_protocol_event(event);
|
||||
if let Some(next_busy) = next_busy {
|
||||
bridge_busy.store(next_busy, Ordering::SeqCst);
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(message) => {
|
||||
let _ = bridge_context.publish_protocol_event(Event::Error {
|
||||
code: protocol::ErrorCode::Internal,
|
||||
message: format!("worker state stream rejected: {message}"),
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
@@ -1493,14 +1502,13 @@ where
|
||||
RuntimeWorkerExecution {
|
||||
handle,
|
||||
shutdown,
|
||||
busy,
|
||||
worker_state,
|
||||
workspace_client,
|
||||
},
|
||||
);
|
||||
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: working_directory.map(|binding| binding.status()),
|
||||
}
|
||||
}
|
||||
@@ -1516,45 +1524,28 @@ impl<F> Drop for WorkerRuntimeExecutionBackend<F> {
|
||||
}
|
||||
}
|
||||
|
||||
fn method_starts_turn(method: &Method) -> bool {
|
||||
matches!(
|
||||
method,
|
||||
Method::Submit { .. }
|
||||
| Method::SubmitTracked { .. }
|
||||
| Method::Notify { auto_run: true, .. }
|
||||
| Method::NotifyTracked { auto_run: true, .. }
|
||||
| Method::Resume
|
||||
| Method::Compact
|
||||
)
|
||||
}
|
||||
|
||||
fn method_can_start_turn_from_status(method: &Method, status: WorkerStatus) -> bool {
|
||||
match method {
|
||||
Method::Resume => matches!(status, WorkerStatus::Idle | WorkerStatus::Paused),
|
||||
_ => status == WorkerStatus::Idle,
|
||||
fn apply_protocol_worker_state(
|
||||
current: &Arc<RwLock<protocol::WorkerStateSnapshot>>,
|
||||
event: &mut Event,
|
||||
) -> Result<bool, String> {
|
||||
let (incoming, replace_stale) = match event {
|
||||
Event::WorkerState { snapshot } => (snapshot, false),
|
||||
Event::Snapshot { state, .. } => (state, true),
|
||||
Event::CommandAcknowledged { acknowledgement } => (&mut acknowledgement.state, true),
|
||||
_ => return Ok(true),
|
||||
};
|
||||
let mut current = current
|
||||
.write()
|
||||
.map_err(|_| "worker state projection lock is poisoned".to_string())?;
|
||||
match protocol::apply_worker_state_snapshot(&mut current, incoming) {
|
||||
Ok(protocol::WorkerStateSnapshotApply::Applied)
|
||||
| Ok(protocol::WorkerStateSnapshotApply::Duplicate) => Ok(true),
|
||||
Ok(protocol::WorkerStateSnapshotApply::Stale) if replace_stale => {
|
||||
*incoming = current.clone();
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExecutionRunState {
|
||||
match status {
|
||||
WorkerStatus::Running => WorkerExecutionRunState::Busy,
|
||||
WorkerStatus::Idle if auto_run => WorkerExecutionRunState::Busy,
|
||||
WorkerStatus::Idle | WorkerStatus::Paused | WorkerStatus::Stopped => {
|
||||
WorkerExecutionRunState::Idle
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
|
||||
match method {
|
||||
Method::Submit { .. }
|
||||
| Method::SubmitTracked { .. }
|
||||
| Method::Notify { auto_run: true, .. }
|
||||
| Method::NotifyTracked { auto_run: true, .. }
|
||||
| Method::Resume
|
||||
| Method::Compact => WorkerExecutionRunState::Busy,
|
||||
Method::Shutdown => WorkerExecutionRunState::Stopped,
|
||||
_ => WorkerExecutionRunState::Idle,
|
||||
Ok(protocol::WorkerStateSnapshotApply::Stale) => Ok(false),
|
||||
Err(error) => Err(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1883,7 +1874,7 @@ where
|
||||
handle: &WorkerExecutionHandle,
|
||||
input: WorkerInput,
|
||||
) -> WorkerExecutionResult {
|
||||
let (worker, busy, _workspace_client) = match self.get_execution(handle) {
|
||||
let (worker, worker_state, _workspace_client) = match self.get_execution(handle) {
|
||||
Ok(execution) => execution,
|
||||
Err(mut result) => {
|
||||
result.operation = WorkerExecutionOperation::Input;
|
||||
@@ -1892,16 +1883,10 @@ where
|
||||
};
|
||||
|
||||
if input.kind == WorkerInputKind::Notify {
|
||||
let status = worker.shared_state.get_status();
|
||||
let accepted_run_state = accepted_notify_run_state(status, true);
|
||||
let claimed_here = status == WorkerStatus::Idle
|
||||
&& busy
|
||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.is_ok();
|
||||
let notification_request_id = input
|
||||
.submission_request_id
|
||||
.unwrap_or_else(protocol::new_submission_request_id);
|
||||
let result = self.send_method(
|
||||
return self.send_method(
|
||||
WorkerExecutionOperation::Input,
|
||||
worker,
|
||||
Method::NotifyTracked {
|
||||
@@ -1912,25 +1897,20 @@ where
|
||||
operation_id: notification_request_id,
|
||||
},
|
||||
},
|
||||
accepted_run_state,
|
||||
);
|
||||
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
||||
{
|
||||
busy.store(false, Ordering::SeqCst);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
let is_user_submit = input.kind == WorkerInputKind::User;
|
||||
let status = worker.shared_state.get_status();
|
||||
let claimed_here = status == WorkerStatus::Idle
|
||||
&& busy
|
||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.is_ok();
|
||||
if !is_user_submit && !claimed_here {
|
||||
return WorkerExecutionResult::busy(
|
||||
if input.kind == WorkerInputKind::Compact {
|
||||
let command = match next_internal_command(&worker_state) {
|
||||
Ok(command) => command,
|
||||
Err(error) => {
|
||||
return WorkerExecutionResult::errored(WorkerExecutionOperation::Input, error);
|
||||
}
|
||||
};
|
||||
return self.send_method(
|
||||
WorkerExecutionOperation::Input,
|
||||
"Worker is already running",
|
||||
worker,
|
||||
Method::Compact { command },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1940,7 +1920,6 @@ where
|
||||
.submission_request_id
|
||||
.filter(|submission_id| !submission_id.trim().is_empty())
|
||||
else {
|
||||
busy.store(false, Ordering::SeqCst);
|
||||
return WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Input,
|
||||
"Runtime user input is missing its internal submission id",
|
||||
@@ -1960,9 +1939,9 @@ where
|
||||
)
|
||||
}
|
||||
WorkerInputKind::Notify => {
|
||||
unreachable!("Notify input is dispatched before the turn-start busy guard")
|
||||
unreachable!("Notify input is dispatched before ordinary input mapping")
|
||||
}
|
||||
WorkerInputKind::Compact => (Method::Compact, None),
|
||||
WorkerInputKind::Compact => unreachable!("compact input is dispatched above"),
|
||||
WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None),
|
||||
WorkerInputKind::RegisterPeer => (
|
||||
Method::RegisterPeer {
|
||||
@@ -1971,40 +1950,18 @@ where
|
||||
None,
|
||||
),
|
||||
};
|
||||
let accepted_run_state = match method {
|
||||
Method::Submit { .. }
|
||||
| Method::SubmitTracked { .. }
|
||||
| Method::Notify { .. }
|
||||
| Method::NotifyTracked { .. }
|
||||
| Method::Compact => WorkerExecutionRunState::Busy,
|
||||
_ => WorkerExecutionRunState::Idle,
|
||||
};
|
||||
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
|
||||
let waits_for_submission_acceptance = submission_request_id.is_some();
|
||||
|
||||
let result = if waits_for_submission_acceptance {
|
||||
if waits_for_submission_acceptance {
|
||||
self.send_submit_and_wait_for_acceptance(
|
||||
WorkerExecutionOperation::Input,
|
||||
worker,
|
||||
method,
|
||||
submission_request_id.expect("Submit must have a submission request id"),
|
||||
accepted_run_state,
|
||||
)
|
||||
} else {
|
||||
self.send_method(
|
||||
WorkerExecutionOperation::Input,
|
||||
worker,
|
||||
method,
|
||||
accepted_run_state,
|
||||
)
|
||||
};
|
||||
if accepted_is_idle
|
||||
|| (claimed_here
|
||||
&& result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
|
||||
{
|
||||
busy.store(false, Ordering::SeqCst);
|
||||
self.send_method(WorkerExecutionOperation::Input, worker, method)
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn upload_file(
|
||||
@@ -2046,10 +2003,7 @@ where
|
||||
}
|
||||
};
|
||||
match worker.delete_uploaded_file(artifact_id) {
|
||||
Ok(_) => WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::DeleteUploadedFile,
|
||||
WorkerExecutionRunState::Idle,
|
||||
),
|
||||
Ok(_) => WorkerExecutionResult::accepted(WorkerExecutionOperation::DeleteUploadedFile),
|
||||
Err(error) => WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::DeleteUploadedFile,
|
||||
format!("uploaded_file_delete_rejected: {error}"),
|
||||
@@ -2062,7 +2016,7 @@ where
|
||||
handle: &WorkerExecutionHandle,
|
||||
method: Method,
|
||||
) -> WorkerExecutionResult {
|
||||
let (worker, busy, _workspace_client) = match self.get_execution(handle) {
|
||||
let (worker, _worker_state, _workspace_client) = match self.get_execution(handle) {
|
||||
Ok(execution) => execution,
|
||||
Err(mut result) => {
|
||||
result.operation = WorkerExecutionOperation::ProtocolMethod;
|
||||
@@ -2070,59 +2024,7 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(auto_run) = match &method {
|
||||
Method::Notify { auto_run, .. } | Method::NotifyTracked { auto_run, .. } => {
|
||||
Some(*auto_run)
|
||||
}
|
||||
_ => None,
|
||||
} {
|
||||
let status = worker.shared_state.get_status();
|
||||
let accepted_run_state = accepted_notify_run_state(status, auto_run);
|
||||
let claimed_here = status == WorkerStatus::Idle
|
||||
&& auto_run
|
||||
&& busy
|
||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.is_ok();
|
||||
let result = self.send_method(
|
||||
WorkerExecutionOperation::ProtocolMethod,
|
||||
worker,
|
||||
method,
|
||||
accepted_run_state,
|
||||
);
|
||||
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
||||
{
|
||||
busy.store(false, Ordering::SeqCst);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
let starts_turn = method_starts_turn(&method);
|
||||
if starts_turn
|
||||
&& (!method_can_start_turn_from_status(&method, worker.shared_state.get_status())
|
||||
|| busy
|
||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.is_err())
|
||||
{
|
||||
return WorkerExecutionResult::busy(
|
||||
WorkerExecutionOperation::ProtocolMethod,
|
||||
"Worker is already running; runtime adapter v0 does not queue protocol methods",
|
||||
);
|
||||
}
|
||||
|
||||
let accepted_run_state = accepted_run_state_for_method(&method);
|
||||
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
|
||||
let result = self.send_method(
|
||||
WorkerExecutionOperation::ProtocolMethod,
|
||||
worker,
|
||||
method,
|
||||
accepted_run_state,
|
||||
);
|
||||
if (starts_turn && accepted_is_idle)
|
||||
|| (starts_turn && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
|
||||
{
|
||||
busy.store(false, Ordering::SeqCst);
|
||||
}
|
||||
result
|
||||
self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method)
|
||||
}
|
||||
|
||||
fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
@@ -2137,7 +2039,7 @@ where
|
||||
);
|
||||
}
|
||||
let execution = match self.workers.lock() {
|
||||
Ok(mut workers) => workers.remove(handle.worker_ref()),
|
||||
Ok(workers) => workers.get(handle.worker_ref()).cloned(),
|
||||
Err(_) => {
|
||||
return WorkerExecutionResult::errored(
|
||||
WorkerExecutionOperation::Stop,
|
||||
@@ -2153,48 +2055,73 @@ where
|
||||
};
|
||||
let artifact_cleanup = execution.handle.clone();
|
||||
let shutdown = execution.shutdown.clone();
|
||||
let command = match next_internal_command(&execution.worker_state) {
|
||||
Ok(command) => command,
|
||||
Err(error) => {
|
||||
return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, error);
|
||||
}
|
||||
};
|
||||
let result = self.send_method(
|
||||
WorkerExecutionOperation::Stop,
|
||||
execution.handle,
|
||||
Method::Shutdown,
|
||||
WorkerExecutionRunState::Stopped,
|
||||
execution.handle.clone(),
|
||||
Method::Shutdown { command },
|
||||
);
|
||||
if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted {
|
||||
return result;
|
||||
}
|
||||
match self.run_on_adapter_runtime(async move {
|
||||
let receiver = shutdown.lock().await.take();
|
||||
if let Some(receiver) = receiver {
|
||||
receiver
|
||||
.await
|
||||
.map_err(|_| "Worker shutdown completion channel closed".to_string())?;
|
||||
let shutdown_wait = self.run_on_adapter_runtime(async move {
|
||||
let mut guard = shutdown.lock().await;
|
||||
let Some(mut receiver) = guard.take() else {
|
||||
return Ok(());
|
||||
};
|
||||
match tokio::time::timeout(Duration::from_secs(5), &mut receiver).await {
|
||||
Ok(Ok(())) => Ok(()),
|
||||
Ok(Err(_)) => Err("Worker shutdown completion channel closed".to_string()),
|
||||
Err(_) => {
|
||||
*guard = Some(receiver);
|
||||
Err("Worker shutdown confirmation timed out; stop remains retryable".into())
|
||||
}
|
||||
Ok(())
|
||||
}) {
|
||||
Ok(()) => match artifact_cleanup.delete_uncommitted_uploaded_files() {
|
||||
Ok(_) => result,
|
||||
Err(error) => WorkerExecutionResult::errored(
|
||||
}
|
||||
});
|
||||
if let Err(message) = shutdown_wait {
|
||||
return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message);
|
||||
}
|
||||
if let Err(error) = artifact_cleanup.delete_uncommitted_uploaded_files() {
|
||||
return WorkerExecutionResult::errored(
|
||||
WorkerExecutionOperation::Stop,
|
||||
format!("uploaded_file_cleanup_failed: {error}"),
|
||||
);
|
||||
}
|
||||
match self.workers.lock() {
|
||||
Ok(mut workers) => {
|
||||
workers.remove(handle.worker_ref());
|
||||
result
|
||||
}
|
||||
Err(_) => WorkerExecutionResult::errored(
|
||||
WorkerExecutionOperation::Stop,
|
||||
"worker adapter registry lock is poisoned after shutdown",
|
||||
),
|
||||
},
|
||||
Err(message) => WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message),
|
||||
}
|
||||
}
|
||||
|
||||
fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
let (worker, _busy, _workspace_client) = match self.get_execution(handle) {
|
||||
let (worker, worker_state, _workspace_client) = match self.get_execution(handle) {
|
||||
Ok(execution) => execution,
|
||||
Err(mut result) => {
|
||||
result.operation = WorkerExecutionOperation::Cancel;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
let command = match next_internal_command(&worker_state) {
|
||||
Ok(command) => command,
|
||||
Err(error) => {
|
||||
return WorkerExecutionResult::errored(WorkerExecutionOperation::Cancel, error);
|
||||
}
|
||||
};
|
||||
self.send_method(
|
||||
WorkerExecutionOperation::Cancel,
|
||||
worker,
|
||||
Method::Cancel,
|
||||
WorkerExecutionRunState::Idle,
|
||||
Method::Cancel { command },
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2259,6 +2186,79 @@ mod tests {
|
||||
use manifest::{Scope, WorkerManifest};
|
||||
use session_store::{LogEntry, WorkerMetadataStore};
|
||||
|
||||
fn test_command() -> WorkerCommandEnvelope {
|
||||
WorkerCommandEnvelope {
|
||||
command_id: 1,
|
||||
expected_execution_generation: 1,
|
||||
expected_worker_state_revision: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn adapter_command(
|
||||
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
|
||||
worker_ref: &WorkerRef,
|
||||
) -> WorkerCommandEnvelope {
|
||||
let workers = backend.workers.lock().unwrap();
|
||||
let state = workers
|
||||
.get(worker_ref)
|
||||
.expect("worker execution")
|
||||
.worker_state
|
||||
.read()
|
||||
.unwrap()
|
||||
.clone();
|
||||
WorkerCommandEnvelope::for_snapshot(state.last_command_id.saturating_add(1), &state)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_bridge_applies_state_and_acknowledgement_monotonically() {
|
||||
let running = protocol::WorkerStateSnapshot {
|
||||
execution_generation: 4,
|
||||
revision: 3,
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)),
|
||||
last_command_id: 2,
|
||||
};
|
||||
let current = Arc::new(RwLock::new(running.clone()));
|
||||
let mut stale = Event::WorkerState {
|
||||
snapshot: protocol::WorkerStateSnapshot {
|
||||
revision: 2,
|
||||
state: protocol::WorkerState::Idle,
|
||||
..running.clone()
|
||||
},
|
||||
};
|
||||
assert!(!apply_protocol_worker_state(¤t, &mut stale).unwrap());
|
||||
assert_eq!(*current.read().unwrap(), running);
|
||||
|
||||
let paused = protocol::WorkerStateSnapshot {
|
||||
revision: 4,
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Paused,
|
||||
)),
|
||||
last_command_id: 3,
|
||||
..running.clone()
|
||||
};
|
||||
let mut acknowledgement = Event::CommandAcknowledged {
|
||||
acknowledgement: protocol::WorkerCommandAcknowledgement {
|
||||
command_id: 3,
|
||||
command: protocol::WorkerCommandKind::Pause,
|
||||
disposition: protocol::WorkerCommandDisposition::Accepted,
|
||||
state: paused.clone(),
|
||||
},
|
||||
};
|
||||
assert!(apply_protocol_worker_state(¤t, &mut acknowledgement).unwrap());
|
||||
assert_eq!(*current.read().unwrap(), paused);
|
||||
|
||||
let mut conflict = Event::WorkerState {
|
||||
snapshot: protocol::WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Idle,
|
||||
..paused.clone()
|
||||
},
|
||||
};
|
||||
assert!(apply_protocol_worker_state(¤t, &mut conflict).is_err());
|
||||
assert_eq!(*current.read().unwrap(), paused);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_prompt_projection_notification_advances_shared_cache() {
|
||||
let cache = WorkspacePromptProjectionCache::default();
|
||||
@@ -2405,46 +2405,6 @@ mod tests {
|
||||
assert_eq!(after_restore_workspace_id.as_deref(), Some("workspace-a"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notify_run_state_allows_running_worker_inbox_delivery() {
|
||||
assert_eq!(
|
||||
accepted_notify_run_state(WorkerStatus::Running, true),
|
||||
WorkerExecutionRunState::Busy
|
||||
);
|
||||
assert_eq!(
|
||||
accepted_notify_run_state(WorkerStatus::Idle, true),
|
||||
WorkerExecutionRunState::Busy
|
||||
);
|
||||
assert_eq!(
|
||||
accepted_notify_run_state(WorkerStatus::Idle, false),
|
||||
WorkerExecutionRunState::Idle
|
||||
);
|
||||
assert_eq!(
|
||||
accepted_notify_run_state(WorkerStatus::Paused, true),
|
||||
WorkerExecutionRunState::Idle
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_turn_claim_accepts_paused_and_idle_but_not_running_status() {
|
||||
assert!(method_can_start_turn_from_status(
|
||||
&Method::Resume,
|
||||
WorkerStatus::Paused
|
||||
));
|
||||
assert!(method_can_start_turn_from_status(
|
||||
&Method::Resume,
|
||||
WorkerStatus::Idle
|
||||
));
|
||||
assert!(!method_can_start_turn_from_status(
|
||||
&Method::Resume,
|
||||
WorkerStatus::Running
|
||||
));
|
||||
assert!(!method_can_start_turn_from_status(
|
||||
&Method::Compact,
|
||||
WorkerStatus::Paused
|
||||
));
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum MockResponse {
|
||||
Complete(Vec<LlmEvent>),
|
||||
@@ -2645,28 +2605,53 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn wait_for_adapter_command(
|
||||
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
|
||||
worker_ref: &WorkerRef,
|
||||
expected_command_id: u64,
|
||||
) {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let observed = {
|
||||
let workers = backend.workers.lock().unwrap();
|
||||
workers
|
||||
.get(worker_ref)
|
||||
.expect("live Worker execution")
|
||||
.worker_state
|
||||
.read()
|
||||
.unwrap()
|
||||
.last_command_id
|
||||
};
|
||||
if observed >= expected_command_id {
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"timed out waiting for adapter command {expected_command_id}; last observed={observed}",
|
||||
);
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_for_adapter_state(
|
||||
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
|
||||
worker_ref: &WorkerRef,
|
||||
expected_status: WorkerStatus,
|
||||
expected_busy: bool,
|
||||
) {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let observed = {
|
||||
let workers = backend.workers.lock().unwrap();
|
||||
let execution = workers.get(worker_ref).expect("live Worker execution");
|
||||
(
|
||||
execution.handle.shared_state.get_status(),
|
||||
execution.busy.load(Ordering::SeqCst),
|
||||
)
|
||||
let projected = execution.worker_state.read().unwrap().catalog_status();
|
||||
(execution.handle.shared_state.catalog_status(), projected)
|
||||
};
|
||||
if observed == (expected_status, expected_busy) {
|
||||
if observed == (expected_status, expected_status) {
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"timed out waiting for adapter state {expected_status:?}, busy={expected_busy}; last observed status={:?}, busy={}",
|
||||
"timed out waiting for adapter state {expected_status:?}; last observed controller={:?}, projected={:?}",
|
||||
observed.0,
|
||||
observed.1,
|
||||
);
|
||||
@@ -3169,13 +3154,19 @@ mod tests {
|
||||
.expect("in-process restore must not bind the overlong Unix socket path");
|
||||
|
||||
assert_eq!(
|
||||
controller.handle.shared_state.get_status(),
|
||||
controller.handle.shared_state.catalog_status(),
|
||||
WorkerStatus::Idle
|
||||
);
|
||||
assert!(!socket_path.exists());
|
||||
assert!(run_dir.join("worker.out.log").is_file());
|
||||
assert!(run_dir.join("worker.err.log").is_file());
|
||||
controller.handle.send(Method::Shutdown).await.unwrap();
|
||||
controller
|
||||
.handle
|
||||
.send(Method::Shutdown {
|
||||
command: test_command(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if let Some(receiver) = controller.shutdown.lock().await.take() {
|
||||
receiver.await.unwrap();
|
||||
}
|
||||
@@ -3289,7 +3280,9 @@ mod tests {
|
||||
backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
handle
|
||||
.send(Method::Shutdown)
|
||||
.send(Method::Shutdown {
|
||||
command: test_command(),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if let Some(receiver) = shutdown.lock().await.take() {
|
||||
@@ -3614,6 +3607,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "ws-server")]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn adapter_resumes_paused_turn_once_and_preserves_idle_not_paused_error() {
|
||||
let hanging_events = || simple_text_events().into_iter().take(2).collect::<Vec<_>>();
|
||||
let client = MockClient::sequential(vec![
|
||||
@@ -3646,62 +3640,91 @@ mod tests {
|
||||
runtime
|
||||
.send_input(&detail.worker_ref, WorkerInput::user("pause and resume"))
|
||||
.expect("start initial turn");
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true);
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running);
|
||||
|
||||
let running_resume = runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
||||
.expect_err("Resume while Running must be rejected");
|
||||
assert!(
|
||||
running_resume
|
||||
.to_string()
|
||||
.contains("does not queue protocol methods"),
|
||||
"unexpected Running Resume error: {running_resume}"
|
||||
);
|
||||
let running_resume = adapter_command(&backend, &detail.worker_ref);
|
||||
runtime
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Resume {
|
||||
command: running_resume,
|
||||
},
|
||||
)
|
||||
.expect("running Resume is forwarded for controller admission");
|
||||
wait_for_adapter_command(&backend, &detail.worker_ref, running_resume.command_id);
|
||||
|
||||
runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Pause)
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Pause {
|
||||
command: adapter_command(&backend, &detail.worker_ref),
|
||||
},
|
||||
)
|
||||
.expect("pause initial turn");
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false);
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused);
|
||||
|
||||
runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Resume {
|
||||
command: adapter_command(&backend, &detail.worker_ref),
|
||||
},
|
||||
)
|
||||
.expect("resume paused turn");
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true);
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running);
|
||||
|
||||
let duplicate_resume = runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
||||
.expect_err("duplicate Resume must be rejected");
|
||||
assert!(
|
||||
duplicate_resume
|
||||
.to_string()
|
||||
.contains("does not queue protocol methods"),
|
||||
"unexpected duplicate Resume error: {duplicate_resume}"
|
||||
);
|
||||
let duplicate_resume = adapter_command(&backend, &detail.worker_ref);
|
||||
runtime
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Resume {
|
||||
command: duplicate_resume,
|
||||
},
|
||||
)
|
||||
.expect("duplicate Resume is forwarded for controller admission");
|
||||
wait_for_adapter_command(&backend, &detail.worker_ref, duplicate_resume.command_id);
|
||||
|
||||
runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Pause)
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Pause {
|
||||
command: adapter_command(&backend, &detail.worker_ref),
|
||||
},
|
||||
)
|
||||
.expect("pause resumed turn");
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false);
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused);
|
||||
runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Resume {
|
||||
command: adapter_command(&backend, &detail.worker_ref),
|
||||
},
|
||||
)
|
||||
.expect("resume paused turn a second time");
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false);
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle);
|
||||
assert_eq!(call_count.load(Ordering::SeqCst), 3);
|
||||
|
||||
let idle_resume = adapter_command(&backend, &detail.worker_ref);
|
||||
runtime
|
||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
||||
.send_protocol_method(
|
||||
&detail.worker_ref,
|
||||
Method::Resume {
|
||||
command: idle_resume,
|
||||
},
|
||||
)
|
||||
.expect("Idle Resume preserves controller NotPaused semantics");
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false);
|
||||
wait_for_adapter_command(&backend, &detail.worker_ref, idle_resume.command_id);
|
||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle);
|
||||
let events = runtime
|
||||
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
|
||||
.expect("read protocol events");
|
||||
assert!(events.iter().any(|event| {
|
||||
matches!(
|
||||
&event.payload,
|
||||
Event::Error {
|
||||
code: protocol::ErrorCode::NotPaused,
|
||||
..
|
||||
}
|
||||
Event::CommandAcknowledged { acknowledgement }
|
||||
if acknowledgement.command == protocol::WorkerCommandKind::Resume
|
||||
&& acknowledgement.disposition
|
||||
== protocol::WorkerCommandDisposition::InvalidState
|
||||
)
|
||||
}));
|
||||
assert_eq!(call_count.load(Ordering::SeqCst), 3);
|
||||
|
||||
+649
-96
@@ -1,3 +1,4 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -16,7 +17,7 @@ use crate::ipc::notify_buffer::NotifyBuffer;
|
||||
use crate::ipc::server::SocketServer;
|
||||
use crate::runtime::dir::RuntimeDir;
|
||||
use crate::segment_log_sink::SegmentLogSink;
|
||||
use crate::shared_state::WorkerSharedState;
|
||||
use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState};
|
||||
use crate::shutdown_after_idle::{
|
||||
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
||||
take_shutdown_request_after_status,
|
||||
@@ -28,7 +29,9 @@ use protocol::{
|
||||
AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent,
|
||||
CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus,
|
||||
CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice,
|
||||
ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, WorkerStatus,
|
||||
ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef,
|
||||
WorkerBusyState, WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope,
|
||||
WorkerCommandKind, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStatus,
|
||||
};
|
||||
use workdir::{
|
||||
CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot,
|
||||
@@ -138,7 +141,7 @@ impl WorkerHandle {
|
||||
let event = Event::Snapshot {
|
||||
session,
|
||||
greeting: self.shared_state.greeting.clone(),
|
||||
status: self.shared_state.get_status(),
|
||||
state: self.shared_state.snapshot(),
|
||||
in_flight,
|
||||
internal_workers: self.spawned_registry.internal_worker_snapshots(),
|
||||
};
|
||||
@@ -178,15 +181,101 @@ impl WorkerHandle {
|
||||
}
|
||||
}
|
||||
|
||||
fn command_admission_disposition(
|
||||
admission: WorkerCommandAdmission,
|
||||
) -> Result<(), WorkerCommandDisposition> {
|
||||
match admission {
|
||||
WorkerCommandAdmission::Accepted => Ok(()),
|
||||
WorkerCommandAdmission::Retry | WorkerCommandAdmission::StaleCommandId => {
|
||||
Err(WorkerCommandDisposition::StaleCommandId)
|
||||
}
|
||||
WorkerCommandAdmission::Conflict => Err(WorkerCommandDisposition::Conflict),
|
||||
WorkerCommandAdmission::ExecutionGenerationMismatch => {
|
||||
Err(WorkerCommandDisposition::StaleExecutionGeneration)
|
||||
}
|
||||
WorkerCommandAdmission::StateRevisionMismatch => {
|
||||
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_command(
|
||||
envelope: WorkerCommandEnvelope,
|
||||
kind: WorkerCommandKind,
|
||||
shared_state: &WorkerSharedState,
|
||||
) -> Result<(), WorkerCommandDisposition> {
|
||||
command_admission_disposition(shared_state.admit_command(envelope, kind, true))
|
||||
}
|
||||
|
||||
fn validate_shutdown_command(
|
||||
envelope: WorkerCommandEnvelope,
|
||||
shared_state: &WorkerSharedState,
|
||||
) -> Result<(), WorkerCommandDisposition> {
|
||||
match shared_state.admit_command(envelope, WorkerCommandKind::Shutdown, false) {
|
||||
WorkerCommandAdmission::Accepted | WorkerCommandAdmission::Retry => Ok(()),
|
||||
admission => command_admission_disposition(admission),
|
||||
}
|
||||
}
|
||||
|
||||
fn acknowledge_command(
|
||||
working_event_tx: &broadcast::Sender<Event>,
|
||||
shared_state: &WorkerSharedState,
|
||||
command_id: u64,
|
||||
command: WorkerCommandKind,
|
||||
disposition: WorkerCommandDisposition,
|
||||
) {
|
||||
shared_state.complete_command(command_id, command, disposition);
|
||||
let _ = working_event_tx.send(Event::CommandAcknowledged {
|
||||
acknowledgement: WorkerCommandAcknowledgement {
|
||||
command_id,
|
||||
command,
|
||||
disposition,
|
||||
state: shared_state.snapshot(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
fn reject_invalid_command_state(
|
||||
working_event_tx: &broadcast::Sender<Event>,
|
||||
shared_state: &WorkerSharedState,
|
||||
envelope: WorkerCommandEnvelope,
|
||||
command: WorkerCommandKind,
|
||||
) {
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
envelope.command_id,
|
||||
command,
|
||||
WorkerCommandDisposition::InvalidState,
|
||||
);
|
||||
}
|
||||
|
||||
async fn set_controller_state(
|
||||
shared_state: &Arc<WorkerSharedState>,
|
||||
runtime_dir: &RuntimeDir,
|
||||
working_event_tx: &broadcast::Sender<Event>,
|
||||
state: WorkerState,
|
||||
) -> protocol::WorkerStateSnapshot {
|
||||
let snapshot = shared_state.transition(state);
|
||||
let _ = runtime_dir.write_status(shared_state).await;
|
||||
let _ = working_event_tx.send(Event::WorkerState {
|
||||
snapshot: snapshot.clone(),
|
||||
});
|
||||
snapshot
|
||||
}
|
||||
|
||||
async fn set_controller_status(
|
||||
shared_state: &Arc<WorkerSharedState>,
|
||||
runtime_dir: &RuntimeDir,
|
||||
working_event_tx: &broadcast::Sender<Event>,
|
||||
status: WorkerStatus,
|
||||
) {
|
||||
shared_state.set_status(status);
|
||||
let _ = runtime_dir.write_status(shared_state).await;
|
||||
let _ = working_event_tx.send(Event::Status { status });
|
||||
let state = match status {
|
||||
WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
|
||||
WorkerStatus::Running => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||
WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
|
||||
};
|
||||
set_controller_state(shared_state, runtime_dir, working_event_tx, state).await;
|
||||
}
|
||||
|
||||
async fn finish_controller_run<C, St>(
|
||||
@@ -660,12 +749,24 @@ impl WorkerController {
|
||||
// === 4. Initial runtime files + WorkerSharedState + WorkerHandle +
|
||||
// SocketServer ===
|
||||
let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default();
|
||||
worker
|
||||
.recover_unfinished_compaction()
|
||||
.await
|
||||
.map_err(|error| std::io::Error::other(error.to_string()))?;
|
||||
let greeting = build_greeting(&worker);
|
||||
let shared_state = Arc::new(WorkerSharedState::new(
|
||||
let execution_generation = runtime_dir
|
||||
.path()
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.and_then(|name| name.parse::<u64>().ok())
|
||||
.filter(|generation| *generation > 0)
|
||||
.unwrap_or(1);
|
||||
let shared_state = Arc::new(WorkerSharedState::new_with_generation(
|
||||
worker.manifest().worker.name.clone(),
|
||||
worker.segment_id(),
|
||||
manifest_toml.clone(),
|
||||
greeting,
|
||||
execution_generation,
|
||||
));
|
||||
if let Some(fs_for_view) = fs_for_view {
|
||||
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view));
|
||||
@@ -1447,8 +1548,9 @@ async fn controller_loop<C, St>(
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
// Top-of-iteration: if an event handler staged a run, fire it
|
||||
let mut deferred_methods = VecDeque::new();
|
||||
|
||||
'controller: loop {
|
||||
// here so the status flip → drive_turn → finish sequence lives
|
||||
// in one place, regardless of which Method caused it.
|
||||
if let Some(run) = pending.take() {
|
||||
@@ -1599,9 +1701,13 @@ async fn controller_loop<C, St>(
|
||||
continue;
|
||||
}
|
||||
|
||||
let method = match method_rx.recv().await {
|
||||
Some(m) => m,
|
||||
let method = if let Some(method) = deferred_methods.pop_front() {
|
||||
method
|
||||
} else {
|
||||
match method_rx.recv().await {
|
||||
Some(method) => method,
|
||||
None => break,
|
||||
}
|
||||
};
|
||||
|
||||
match method {
|
||||
@@ -1799,7 +1905,7 @@ async fn controller_loop<C, St>(
|
||||
expected_revision,
|
||||
expected_head_id,
|
||||
} => {
|
||||
if shared_state.get_status() != WorkerStatus::Idle {
|
||||
if shared_state.catalog_status() != WorkerStatus::Idle {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(),
|
||||
@@ -1826,88 +1932,266 @@ async fn controller_loop<C, St>(
|
||||
}
|
||||
}
|
||||
}
|
||||
Method::Resume => {
|
||||
if shared_state.get_status() != WorkerStatus::Paused {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::NotPaused,
|
||||
message: "Worker is not paused".into(),
|
||||
});
|
||||
Method::Resume { command } => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Resume, &shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Resume,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
pending = Some(PendingRun::Resume);
|
||||
if !matches!(
|
||||
shared_state.snapshot().state,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused))
|
||||
) {
|
||||
reject_invalid_command_state(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Resume,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
Method::Cancel => match shared_state.get_status() {
|
||||
WorkerStatus::Paused => match worker.cancel_paused_turn() {
|
||||
Ok(()) => {
|
||||
worker.clear_in_flight_events();
|
||||
set_controller_status(
|
||||
set_controller_state(
|
||||
&shared_state,
|
||||
&runtime_dir,
|
||||
&working_event_tx,
|
||||
WorkerStatus::Idle,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||
)
|
||||
.await;
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Resume,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
pending = Some(PendingRun::Resume);
|
||||
}
|
||||
|
||||
Method::Cancel { command } => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Cancel, &shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Cancel,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !matches!(
|
||||
shared_state.snapshot().state,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused))
|
||||
) {
|
||||
reject_invalid_command_state(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Cancel,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
set_controller_state(
|
||||
&shared_state,
|
||||
&runtime_dir,
|
||||
&working_event_tx,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
|
||||
)
|
||||
.await;
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Cancel,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
match worker.cancel_paused_turn() {
|
||||
Ok(()) => {
|
||||
worker.clear_in_flight_events();
|
||||
set_controller_state(
|
||||
&shared_state,
|
||||
&runtime_dir,
|
||||
&working_event_tx,
|
||||
WorkerState::Idle,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: worker_error_code(&error),
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
},
|
||||
WorkerStatus::Idle | WorkerStatus::Stopped => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::NotRunning,
|
||||
message: "Worker is not running".into(),
|
||||
});
|
||||
}
|
||||
WorkerStatus::Running => {
|
||||
// Running turns receive Cancel through drive_turn; this is
|
||||
// only reachable across a defensive race window.
|
||||
let _ = cancel_tx.try_send(());
|
||||
}
|
||||
},
|
||||
|
||||
Method::Pause => {
|
||||
// Already paused → idempotent no-op. Otherwise the
|
||||
// Worker is Idle (Running turns go through `drive_turn`,
|
||||
// not this outer match), so there is nothing to pause.
|
||||
if shared_state.get_status() != WorkerStatus::Paused {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::NotRunning,
|
||||
message: "Worker is not running".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Method::Compact => match shared_state.get_status() {
|
||||
WorkerStatus::Idle => {
|
||||
if let Err(error) = worker.manual_compact().await {
|
||||
set_controller_state(
|
||||
&shared_state,
|
||||
&runtime_dir,
|
||||
&working_event_tx,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
|
||||
)
|
||||
.await;
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: worker_error_code(&error),
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
WorkerStatus::Paused => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: "Cannot compact while the Worker is paused; resume or start a fresh turn first"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
WorkerStatus::Running | WorkerStatus::Stopped => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::AlreadyRunning,
|
||||
message:
|
||||
"Worker is already executing a turn; compact can only run while idle"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
Method::ListRewindTargets => match shared_state.get_status() {
|
||||
Method::Pause { command } => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Pause, &shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Pause,
|
||||
disposition,
|
||||
);
|
||||
} else {
|
||||
reject_invalid_command_state(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Pause,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Method::Compact { command } => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Compact, &shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Compact,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !matches!(shared_state.snapshot().state, WorkerState::Idle) {
|
||||
reject_invalid_command_state(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Compact,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
set_controller_state(
|
||||
&shared_state,
|
||||
&runtime_dir,
|
||||
&working_event_tx,
|
||||
WorkerState::Busy(WorkerBusyState::Maintenance(
|
||||
WorkerMaintenanceState::Compacting,
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Compact,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
|
||||
let mut shutdown_after_compaction = false;
|
||||
let result = {
|
||||
let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx));
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut compact => break result,
|
||||
method = method_rx.recv() => {
|
||||
match method {
|
||||
Some(Method::Cancel { command }) => {
|
||||
if let Err(disposition) = validate_command(
|
||||
command,
|
||||
WorkerCommandKind::Cancel,
|
||||
&shared_state,
|
||||
) {
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Cancel,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Cancel,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
let _ = cancel_tx.send(true);
|
||||
}
|
||||
Some(Method::Shutdown { command }) => {
|
||||
if let Err(disposition) =
|
||||
validate_shutdown_command(command, &shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Shutdown,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
shutdown_after_compaction = true;
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Shutdown,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
let _ = cancel_tx.send(true);
|
||||
}
|
||||
Some(method) => deferred_methods.push_back(method),
|
||||
None => {
|
||||
shutdown_after_compaction = true;
|
||||
let _ = cancel_tx.send(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if !matches!(
|
||||
result,
|
||||
Err(WorkerError::Store(_))
|
||||
| Err(WorkerError::WorkerStore(_))
|
||||
| Err(WorkerError::InvalidState(_))
|
||||
) {
|
||||
set_controller_state(
|
||||
&shared_state,
|
||||
&runtime_dir,
|
||||
&working_event_tx,
|
||||
WorkerState::Idle,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
if let Err(error) = result {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: worker_error_code(&error),
|
||||
message: error.to_string(),
|
||||
});
|
||||
}
|
||||
if shutdown_after_compaction {
|
||||
let _ = working_event_tx.send(Event::Shutdown);
|
||||
break 'controller;
|
||||
}
|
||||
}
|
||||
|
||||
Method::ListRewindTargets => match shared_state.catalog_status() {
|
||||
WorkerStatus::Idle | WorkerStatus::Paused => {
|
||||
emit_rewind_targets(&worker, &working_event_tx)
|
||||
}
|
||||
@@ -1923,7 +2207,7 @@ async fn controller_loop<C, St>(
|
||||
Method::RewindTo {
|
||||
target,
|
||||
expected_head_entries,
|
||||
} => match shared_state.get_status() {
|
||||
} => match shared_state.catalog_status() {
|
||||
WorkerStatus::Idle => {
|
||||
if apply_rewind(
|
||||
&mut worker,
|
||||
@@ -1934,10 +2218,8 @@ async fn controller_loop<C, St>(
|
||||
.await
|
||||
{
|
||||
worker.clear_in_flight_events();
|
||||
shared_state.set_status(WorkerStatus::Idle);
|
||||
let _ = working_event_tx.send(Event::Status {
|
||||
status: WorkerStatus::Idle,
|
||||
});
|
||||
let snapshot = shared_state.transition(WorkerState::Idle);
|
||||
let _ = working_event_tx.send(Event::WorkerState { snapshot });
|
||||
}
|
||||
}
|
||||
WorkerStatus::Paused => {
|
||||
@@ -1956,7 +2238,26 @@ async fn controller_loop<C, St>(
|
||||
}
|
||||
},
|
||||
|
||||
Method::Shutdown => {
|
||||
Method::Shutdown { command } => {
|
||||
// Shutdown ignores the state-revision fence but remains bound to the
|
||||
// current execution generation and command payload identity.
|
||||
if let Err(disposition) = validate_shutdown_command(command, &shared_state) {
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Shutdown,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
acknowledge_command(
|
||||
&working_event_tx,
|
||||
&shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Shutdown,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
let _ = working_event_tx.send(Event::Shutdown);
|
||||
break;
|
||||
}
|
||||
@@ -2038,7 +2339,7 @@ async fn controller_loop<C, St>(
|
||||
// Auto-kick a turn if the Worker is idle so the
|
||||
// notification is not stranded. Matches the
|
||||
// `Method::Notify` idle path.
|
||||
if shared_state.get_status() == WorkerStatus::Idle {
|
||||
if shared_state.catalog_status() == WorkerStatus::Idle {
|
||||
pending = Some(PendingRun::RunForNotification {
|
||||
invoke_kind: protocol::InvokeKind::WorkerEvent,
|
||||
notification_request_id: None,
|
||||
@@ -2294,15 +2595,115 @@ where
|
||||
}
|
||||
method = method_rx.recv(), if input_commit.is_none() => {
|
||||
match method {
|
||||
Some(Method::Cancel) => {
|
||||
Some(Method::Cancel { command }) => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Cancel, shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Cancel,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !matches!(
|
||||
shared_state.snapshot().state,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||
) {
|
||||
reject_invalid_command_state(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Cancel,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
set_controller_state(
|
||||
shared_state,
|
||||
runtime_dir,
|
||||
working_event_tx,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
|
||||
)
|
||||
.await;
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Cancel,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
let _ = cancel_tx.try_send(());
|
||||
}
|
||||
Some(Method::Pause) => {
|
||||
Some(Method::Pause { command }) => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Pause, shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Pause,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if !matches!(
|
||||
shared_state.snapshot().state,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||
) {
|
||||
reject_invalid_command_state(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Pause,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
pause_requested = true;
|
||||
set_controller_state(
|
||||
shared_state,
|
||||
runtime_dir,
|
||||
working_event_tx,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Pausing)),
|
||||
)
|
||||
.await;
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Pause,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
let _ = pause_tx.try_send(());
|
||||
}
|
||||
Some(Method::Shutdown) => {
|
||||
Some(Method::Shutdown { command }) => {
|
||||
if let Err(disposition) = validate_shutdown_command(command, shared_state) {
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Shutdown,
|
||||
disposition,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
shutdown_requested = true;
|
||||
set_controller_state(
|
||||
shared_state,
|
||||
runtime_dir,
|
||||
working_event_tx,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
|
||||
)
|
||||
.await;
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Shutdown,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
let _ = cancel_tx.try_send(());
|
||||
}
|
||||
Some(Method::Submit {
|
||||
@@ -2368,7 +2769,27 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(Method::Resume | Method::ContinuePending { .. }) => {
|
||||
Some(Method::Resume { command }) => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Resume, shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Resume,
|
||||
disposition,
|
||||
);
|
||||
} else {
|
||||
reject_invalid_command_state(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Resume,
|
||||
);
|
||||
}
|
||||
}
|
||||
Some(Method::ContinuePending { .. }) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::AlreadyRunning,
|
||||
message: "Worker is already executing a turn".into(),
|
||||
@@ -2408,7 +2829,27 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => {
|
||||
Some(Method::Compact { command }) => {
|
||||
if let Err(disposition) =
|
||||
validate_command(command, WorkerCommandKind::Compact, shared_state)
|
||||
{
|
||||
acknowledge_command(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command.command_id,
|
||||
WorkerCommandKind::Compact,
|
||||
disposition,
|
||||
);
|
||||
} else {
|
||||
reject_invalid_command_state(
|
||||
working_event_tx,
|
||||
shared_state,
|
||||
command,
|
||||
WorkerCommandKind::Compact,
|
||||
);
|
||||
}
|
||||
}
|
||||
Some(Method::ListRewindTargets | Method::RewindTo { .. }) => {
|
||||
let _ = working_event_tx.send(Event::Error {
|
||||
code: ErrorCode::AlreadyRunning,
|
||||
message: "Worker is already executing a turn; rewind/compact can only run while idle or paused"
|
||||
@@ -2511,7 +2952,7 @@ where
|
||||
}
|
||||
None => {
|
||||
let _ = cancel_tx.try_send(());
|
||||
shared_state.set_status(WorkerStatus::Idle);
|
||||
shared_state.transition(WorkerState::Idle);
|
||||
return (WorkerStatus::Idle, false, false);
|
||||
}
|
||||
}
|
||||
@@ -2887,7 +3328,7 @@ mod tests {
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
@@ -2943,9 +3384,17 @@ mod tests {
|
||||
async fn pause_waits_for_run_boundary_and_uses_safe_pause_channel() {
|
||||
let mut env = make_env().await;
|
||||
let method_tx = env._method_tx.clone();
|
||||
env.shared_state
|
||||
.transition(WorkerState::Busy(WorkerBusyState::Run(
|
||||
WorkerRunState::Running,
|
||||
)));
|
||||
let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot());
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
method_tx.send(Method::Pause).await.expect("send pause");
|
||||
method_tx
|
||||
.send(Method::Pause { command })
|
||||
.await
|
||||
.expect("send pause");
|
||||
});
|
||||
|
||||
let worker_future = async {
|
||||
@@ -3218,8 +3667,13 @@ mod tests {
|
||||
async fn compact_method_is_rejected_while_running() {
|
||||
let mut env = make_env().await;
|
||||
let mut events = env.working_event_tx.subscribe();
|
||||
env.shared_state
|
||||
.transition(WorkerState::Busy(WorkerBusyState::Run(
|
||||
WorkerRunState::Running,
|
||||
)));
|
||||
let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot());
|
||||
env._method_tx
|
||||
.send(Method::Compact)
|
||||
.send(Method::Compact { command })
|
||||
.await
|
||||
.expect("send compact");
|
||||
|
||||
@@ -3252,14 +3706,113 @@ mod tests {
|
||||
.expect("event timeout")
|
||||
.expect("event");
|
||||
match event {
|
||||
Event::Error { code, message } => {
|
||||
assert_eq!(code, ErrorCode::AlreadyRunning);
|
||||
assert!(message.contains("compact"), "got message: {message}");
|
||||
Event::CommandAcknowledged { acknowledgement } => {
|
||||
assert_eq!(acknowledgement.command, WorkerCommandKind::Compact);
|
||||
assert_eq!(
|
||||
acknowledgement.disposition,
|
||||
WorkerCommandDisposition::InvalidState
|
||||
);
|
||||
assert!(matches!(
|
||||
acknowledgement.state.state,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||
));
|
||||
}
|
||||
other => panic!("expected compact rejection error, got {other:?}"),
|
||||
other => panic!("expected compact rejection acknowledgement, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_admission_rejects_stale_generation_revision_and_order() {
|
||||
let shared = WorkerSharedState::new_with_generation(
|
||||
"worker".into(),
|
||||
session_store::new_segment_id(),
|
||||
String::new(),
|
||||
protocol::Greeting {
|
||||
worker_name: "worker".into(),
|
||||
cwd: "/tmp".into(),
|
||||
provider: "test".into(),
|
||||
model: "test".into(),
|
||||
scope_summary: String::new(),
|
||||
tools: Vec::new(),
|
||||
context_window: 1,
|
||||
context_tokens: 0,
|
||||
},
|
||||
9,
|
||||
);
|
||||
assert_eq!(
|
||||
validate_command(
|
||||
WorkerCommandEnvelope {
|
||||
command_id: 1,
|
||||
expected_execution_generation: 8,
|
||||
expected_worker_state_revision: 0,
|
||||
},
|
||||
WorkerCommandKind::Pause,
|
||||
&shared,
|
||||
),
|
||||
Err(WorkerCommandDisposition::StaleExecutionGeneration)
|
||||
);
|
||||
assert_eq!(
|
||||
validate_command(
|
||||
WorkerCommandEnvelope {
|
||||
command_id: 2,
|
||||
expected_execution_generation: 9,
|
||||
expected_worker_state_revision: 1,
|
||||
},
|
||||
WorkerCommandKind::Pause,
|
||||
&shared,
|
||||
),
|
||||
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
|
||||
);
|
||||
assert!(
|
||||
validate_command(
|
||||
WorkerCommandEnvelope {
|
||||
command_id: 1,
|
||||
expected_execution_generation: 9,
|
||||
expected_worker_state_revision: 0,
|
||||
},
|
||||
WorkerCommandKind::Pause,
|
||||
&shared,
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert_eq!(
|
||||
validate_command(
|
||||
WorkerCommandEnvelope {
|
||||
command_id: 1,
|
||||
expected_execution_generation: 9,
|
||||
expected_worker_state_revision: 0,
|
||||
},
|
||||
WorkerCommandKind::Pause,
|
||||
&shared,
|
||||
),
|
||||
Err(WorkerCommandDisposition::StaleCommandId)
|
||||
);
|
||||
assert_eq!(
|
||||
validate_command(
|
||||
WorkerCommandEnvelope {
|
||||
command_id: 1,
|
||||
expected_execution_generation: 9,
|
||||
expected_worker_state_revision: 0,
|
||||
},
|
||||
WorkerCommandKind::Cancel,
|
||||
&shared,
|
||||
),
|
||||
Err(WorkerCommandDisposition::Conflict)
|
||||
);
|
||||
assert!(
|
||||
validate_command(
|
||||
WorkerCommandEnvelope {
|
||||
command_id: 2,
|
||||
expected_execution_generation: 9,
|
||||
expected_worker_state_revision: 1,
|
||||
},
|
||||
WorkerCommandKind::Pause,
|
||||
&shared,
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn controller_shutdown_orders_child_cleanup_before_workdir_close() {
|
||||
let source = include_str!("controller.rs");
|
||||
|
||||
@@ -779,10 +779,10 @@ async fn probe_socket(socket_path: &Path) -> LiveInfo {
|
||||
loop {
|
||||
match tokio::time::timeout(PROBE_TIMEOUT, reader.next::<Event>()).await {
|
||||
Ok(Ok(Some(Event::Snapshot {
|
||||
status: snapshot_status,
|
||||
state: snapshot_state,
|
||||
..
|
||||
}))) => {
|
||||
status = Some(snapshot_status);
|
||||
status = Some(snapshot_state.catalog_status());
|
||||
break;
|
||||
}
|
||||
Ok(Ok(Some(Event::Alert(_)))) => continue,
|
||||
@@ -1507,7 +1507,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
@@ -1543,7 +1543,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
@@ -1638,7 +1638,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
@@ -1665,7 +1665,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
@@ -1773,7 +1773,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Paused,
|
||||
state: WorkerStatus::Paused.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
@@ -1827,7 +1827,7 @@ mod tests {
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
|
||||
@@ -295,6 +295,38 @@ impl InternalWorkerSessionStatus {
|
||||
}
|
||||
}
|
||||
|
||||
fn send_internal_worker_state(
|
||||
event_tx: &broadcast::Sender<Event>,
|
||||
state_revision: &std::sync::atomic::AtomicU64,
|
||||
status: InternalWorkerSessionStatus,
|
||||
) {
|
||||
let state = match status {
|
||||
InternalWorkerSessionStatus::Idle
|
||||
| InternalWorkerSessionStatus::Stopped
|
||||
| InternalWorkerSessionStatus::Failed => protocol::WorkerState::Idle,
|
||||
InternalWorkerSessionStatus::Paused => protocol::WorkerState::Busy(
|
||||
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Paused),
|
||||
),
|
||||
InternalWorkerSessionStatus::Running => protocol::WorkerState::Busy(
|
||||
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running),
|
||||
),
|
||||
InternalWorkerSessionStatus::Stopping => protocol::WorkerState::Busy(
|
||||
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Cancelling),
|
||||
),
|
||||
};
|
||||
let revision = state_revision
|
||||
.fetch_add(1, std::sync::atomic::Ordering::AcqRel)
|
||||
.saturating_add(1);
|
||||
let _ = event_tx.send(Event::WorkerState {
|
||||
snapshot: protocol::WorkerStateSnapshot {
|
||||
execution_generation: 1,
|
||||
revision,
|
||||
last_command_id: 0,
|
||||
state,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
fn classify_internal_turn_result(
|
||||
result: Result<WorkerRunResult, WorkerError>,
|
||||
) -> (InternalWorkerSessionStatus, Option<String>) {
|
||||
@@ -351,6 +383,7 @@ pub(crate) struct InternalWorkerSessionSnapshot {
|
||||
pub(crate) struct InternalWorkerSessionHandle {
|
||||
command_tx: tokio::sync::mpsc::Sender<InternalWorkerSessionCommand>,
|
||||
status: Arc<std::sync::atomic::AtomicU8>,
|
||||
state_revision: Arc<std::sync::atomic::AtomicU64>,
|
||||
store: EphemeralSessionStore,
|
||||
session_id: SessionId,
|
||||
segment_id: SegmentId,
|
||||
@@ -400,6 +433,10 @@ impl InternalWorkerSessionHandle {
|
||||
self.in_flight.text_delta(block_id, text.to_owned());
|
||||
}
|
||||
|
||||
fn emit_worker_state(&self, status: InternalWorkerSessionStatus) {
|
||||
send_internal_worker_state(&self.event_tx, &self.state_revision, status);
|
||||
}
|
||||
|
||||
pub(crate) fn protocol_snapshot(&self) -> InternalWorkerSessionSnapshot {
|
||||
let (entries, in_flight) = {
|
||||
let guard = self.in_flight.snapshot_guard();
|
||||
@@ -473,9 +510,7 @@ impl InternalWorkerSessionHandle {
|
||||
});
|
||||
return Err(InternalWorkerSessionError::Unavailable);
|
||||
}
|
||||
let _ = self.event_tx.send(Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
});
|
||||
self.emit_worker_state(InternalWorkerSessionStatus::Running);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -770,11 +805,13 @@ pub(crate) async fn prepare_internal_worker_session(
|
||||
let status = Arc::new(std::sync::atomic::AtomicU8::new(
|
||||
InternalWorkerSessionStatus::Idle.encode(),
|
||||
));
|
||||
let state_revision = Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||
let state_changed = Arc::new(tokio::sync::Notify::new());
|
||||
let last_error = Arc::new(Mutex::new(None));
|
||||
let handle = InternalWorkerSessionHandle {
|
||||
command_tx,
|
||||
status: status.clone(),
|
||||
state_revision: state_revision.clone(),
|
||||
store,
|
||||
session_id,
|
||||
segment_id,
|
||||
@@ -810,19 +847,11 @@ pub(crate) async fn prepare_internal_worker_session(
|
||||
message,
|
||||
});
|
||||
}
|
||||
let protocol_status = match turn_status {
|
||||
InternalWorkerSessionStatus::Idle => WorkerStatus::Idle,
|
||||
InternalWorkerSessionStatus::Paused => WorkerStatus::Paused,
|
||||
InternalWorkerSessionStatus::Stopped
|
||||
| InternalWorkerSessionStatus::Failed => WorkerStatus::Stopped,
|
||||
InternalWorkerSessionStatus::Running
|
||||
| InternalWorkerSessionStatus::Stopping => {
|
||||
unreachable!("run completion cannot remain active")
|
||||
}
|
||||
};
|
||||
let _ = event_tx.send(Event::Status {
|
||||
status: protocol_status,
|
||||
});
|
||||
send_internal_worker_state(
|
||||
&event_tx,
|
||||
&state_revision,
|
||||
turn_status,
|
||||
);
|
||||
if let Some(callback) = &on_turn_end {
|
||||
callback(turn_status);
|
||||
}
|
||||
@@ -864,9 +893,11 @@ pub(crate) async fn prepare_internal_worker_session(
|
||||
InternalWorkerSessionStatus::Stopped.encode(),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
let _ = event_tx.send(Event::Status {
|
||||
status: WorkerStatus::Stopped,
|
||||
});
|
||||
send_internal_worker_state(
|
||||
&event_tx,
|
||||
&state_revision,
|
||||
InternalWorkerSessionStatus::Stopped,
|
||||
);
|
||||
let _ = event_tx.send(Event::Shutdown);
|
||||
state_changed.notify_waiters();
|
||||
if let Some(done) = stop_done {
|
||||
@@ -1118,6 +1149,7 @@ pub(crate) fn test_internal_worker_session(
|
||||
status: Arc::new(std::sync::atomic::AtomicU8::new(
|
||||
InternalWorkerSessionStatus::Idle.encode(),
|
||||
)),
|
||||
state_revision: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
store,
|
||||
session_id,
|
||||
segment_id,
|
||||
|
||||
@@ -197,7 +197,6 @@ pub fn default_base() -> Result<PathBuf, io::Error> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::shared_state::WorkerSharedState;
|
||||
use protocol::WorkerStatus;
|
||||
|
||||
fn test_state() -> WorkerSharedState {
|
||||
WorkerSharedState::new(
|
||||
@@ -247,7 +246,9 @@ mod tests {
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
let state = test_state();
|
||||
|
||||
state.set_status(WorkerStatus::Running);
|
||||
state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)));
|
||||
rt.write_status(&state).await.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
|
||||
|
||||
@@ -1,28 +1,49 @@
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{
|
||||
OnceLock, RwLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
use protocol::WorkerStatus;
|
||||
use protocol::{
|
||||
WorkerBusyState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind,
|
||||
WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus,
|
||||
};
|
||||
use serde_json::json;
|
||||
use session_store::SegmentId;
|
||||
|
||||
use crate::fs_view::WorkerFsView;
|
||||
|
||||
const COMPLETED_COMMAND_RETENTION: usize = 256;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct AcceptedWorkerCommand {
|
||||
envelope: WorkerCommandEnvelope,
|
||||
kind: WorkerCommandKind,
|
||||
disposition: Option<WorkerCommandDisposition>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum WorkerCommandAdmission {
|
||||
Accepted,
|
||||
Retry,
|
||||
Conflict,
|
||||
StaleCommandId,
|
||||
ExecutionGenerationMismatch,
|
||||
StateRevisionMismatch,
|
||||
}
|
||||
|
||||
/// Shared state between WorkerController and runtime directory.
|
||||
///
|
||||
/// Controller updates this in-memory; RuntimeDir writes the status
|
||||
/// snapshot to disk. Wrapped in `Arc` for sharing.
|
||||
///
|
||||
/// History and typed user-segment mirrors used to live here so the
|
||||
/// IPC layer could answer `Method::GetHistory`. Those reads now go
|
||||
/// directly through the session-log sink (`Event::Snapshot` +
|
||||
/// live events), so this struct holds only status, identity,
|
||||
/// greeting, and filesystem completion lookup hubs.
|
||||
/// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime
|
||||
/// catalog status remains a separate lifecycle projection because `Stopped`
|
||||
/// describes the execution handle rather than a live controller state.
|
||||
pub struct WorkerSharedState {
|
||||
pub worker_name: String,
|
||||
pub segment_id: SegmentId,
|
||||
pub manifest_toml: String,
|
||||
pub greeting: protocol::Greeting,
|
||||
pub status: RwLock<WorkerStatus>,
|
||||
state: RwLock<WorkerStateSnapshot>,
|
||||
accepted_commands: RwLock<VecDeque<AcceptedWorkerCommand>>,
|
||||
/// Worker-from-the-inside view of the filesystem. Set once in
|
||||
/// `WorkerController::start` after the local WorkdirSession provider is
|
||||
/// materialised, and read from the IPC server layer to answer
|
||||
@@ -38,13 +59,24 @@ impl WorkerSharedState {
|
||||
segment_id: SegmentId,
|
||||
manifest_toml: String,
|
||||
greeting: protocol::Greeting,
|
||||
) -> Self {
|
||||
Self::new_with_generation(worker_name, segment_id, manifest_toml, greeting, 1)
|
||||
}
|
||||
|
||||
pub fn new_with_generation(
|
||||
worker_name: String,
|
||||
segment_id: SegmentId,
|
||||
manifest_toml: String,
|
||||
greeting: protocol::Greeting,
|
||||
execution_generation: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
worker_name,
|
||||
segment_id,
|
||||
manifest_toml,
|
||||
greeting,
|
||||
status: RwLock::new(WorkerStatus::Idle),
|
||||
state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)),
|
||||
accepted_commands: RwLock::new(VecDeque::new()),
|
||||
fs_view: OnceLock::new(),
|
||||
flow_transition_enabled: AtomicBool::new(false),
|
||||
}
|
||||
@@ -70,21 +102,140 @@ impl WorkerSharedState {
|
||||
self.flow_transition_enabled.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn set_status(&self, status: WorkerStatus) {
|
||||
if let Ok(mut s) = self.status.write() {
|
||||
*s = status;
|
||||
pub fn transition(&self, state: WorkerState) -> WorkerStateSnapshot {
|
||||
let mut snapshot = self
|
||||
.state
|
||||
.write()
|
||||
.expect("worker state lock poisoned; refusing an inferred fallback state");
|
||||
if snapshot.state != state {
|
||||
snapshot.revision = snapshot.revision.saturating_add(1);
|
||||
snapshot.state = state;
|
||||
}
|
||||
snapshot.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn admit_command(
|
||||
&self,
|
||||
envelope: WorkerCommandEnvelope,
|
||||
kind: WorkerCommandKind,
|
||||
require_state_revision: bool,
|
||||
) -> WorkerCommandAdmission {
|
||||
let mut snapshot = self
|
||||
.state
|
||||
.write()
|
||||
.expect("worker state lock poisoned; refusing command admission");
|
||||
let mut accepted = self
|
||||
.accepted_commands
|
||||
.write()
|
||||
.expect("worker command ledger lock poisoned; refusing command admission");
|
||||
if let Some(existing) = accepted
|
||||
.iter()
|
||||
.find(|accepted| accepted.envelope.command_id == envelope.command_id)
|
||||
{
|
||||
return if existing.envelope == envelope && existing.kind == kind {
|
||||
WorkerCommandAdmission::Retry
|
||||
} else {
|
||||
WorkerCommandAdmission::Conflict
|
||||
};
|
||||
}
|
||||
if envelope.expected_execution_generation != snapshot.execution_generation {
|
||||
return WorkerCommandAdmission::ExecutionGenerationMismatch;
|
||||
}
|
||||
if require_state_revision && envelope.expected_worker_state_revision != snapshot.revision {
|
||||
return WorkerCommandAdmission::StateRevisionMismatch;
|
||||
}
|
||||
if envelope.command_id <= snapshot.last_command_id {
|
||||
return WorkerCommandAdmission::StaleCommandId;
|
||||
}
|
||||
|
||||
snapshot.last_command_id = envelope.command_id;
|
||||
snapshot.revision = snapshot.revision.saturating_add(1);
|
||||
accepted.push_back(AcceptedWorkerCommand {
|
||||
envelope,
|
||||
kind,
|
||||
disposition: None,
|
||||
});
|
||||
WorkerCommandAdmission::Accepted
|
||||
}
|
||||
|
||||
pub(crate) fn complete_command(
|
||||
&self,
|
||||
command_id: u64,
|
||||
kind: WorkerCommandKind,
|
||||
disposition: WorkerCommandDisposition,
|
||||
) {
|
||||
if !matches!(
|
||||
disposition,
|
||||
WorkerCommandDisposition::Accepted | WorkerCommandDisposition::InvalidState
|
||||
) {
|
||||
return;
|
||||
}
|
||||
let mut accepted = self
|
||||
.accepted_commands
|
||||
.write()
|
||||
.expect("worker command ledger lock poisoned; refusing command completion");
|
||||
if let Some(command) = accepted
|
||||
.iter_mut()
|
||||
.find(|command| command.envelope.command_id == command_id && command.kind == kind)
|
||||
{
|
||||
command.disposition.get_or_insert(disposition);
|
||||
}
|
||||
while accepted
|
||||
.iter()
|
||||
.filter(|command| command.disposition.is_some())
|
||||
.count()
|
||||
> COMPLETED_COMMAND_RETENTION
|
||||
{
|
||||
let Some(index) = accepted
|
||||
.iter()
|
||||
.position(|command| command.disposition.is_some())
|
||||
else {
|
||||
break;
|
||||
};
|
||||
accepted.remove(index);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_status(&self) -> WorkerStatus {
|
||||
self.status.read().map(|s| *s).unwrap_or(WorkerStatus::Idle)
|
||||
#[cfg(test)]
|
||||
pub(crate) fn command_result(
|
||||
&self,
|
||||
command_id: u64,
|
||||
) -> Option<Option<WorkerCommandDisposition>> {
|
||||
self.accepted_commands
|
||||
.read()
|
||||
.expect("worker command ledger lock poisoned")
|
||||
.iter()
|
||||
.find(|command| command.envelope.command_id == command_id)
|
||||
.map(|command| command.disposition)
|
||||
}
|
||||
|
||||
/// Serialize status as JSON.
|
||||
pub fn snapshot(&self) -> WorkerStateSnapshot {
|
||||
self.state
|
||||
.read()
|
||||
.expect("worker state lock poisoned; refusing an inferred fallback state")
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Runtime catalog projection. This must not be used as live command
|
||||
/// admission authority.
|
||||
pub fn catalog_status(&self) -> WorkerStatus {
|
||||
match self.snapshot().state {
|
||||
WorkerState::Idle => WorkerStatus::Idle,
|
||||
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused,
|
||||
WorkerState::Busy(WorkerBusyState::Run(_))
|
||||
| WorkerState::Busy(WorkerBusyState::Maintenance(WorkerMaintenanceState::Compacting)) => {
|
||||
WorkerStatus::Running
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialize the runtime-directory lifecycle projection as JSON while
|
||||
/// retaining the full state snapshot for diagnostics and reconnects.
|
||||
pub fn status_json(&self) -> String {
|
||||
let status = self.get_status();
|
||||
let snapshot = self.snapshot();
|
||||
json!({
|
||||
"state": status,
|
||||
"state": self.catalog_status(),
|
||||
"worker_state": snapshot,
|
||||
"segment_id": self.segment_id.to_string(),
|
||||
"worker_name": self.worker_name,
|
||||
})
|
||||
@@ -97,11 +248,12 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_state() -> WorkerSharedState {
|
||||
WorkerSharedState::new(
|
||||
WorkerSharedState::new_with_generation(
|
||||
"test-worker".into(),
|
||||
session_store::new_segment_id(),
|
||||
"[engine]\nname = \"test-worker\"".into(),
|
||||
test_greeting(),
|
||||
7,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -119,36 +271,82 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initial_status_is_idle() {
|
||||
fn initial_snapshot_is_idle() {
|
||||
let state = test_state();
|
||||
assert_eq!(state.get_status(), WorkerStatus::Idle);
|
||||
assert_eq!(state.snapshot(), WorkerStateSnapshot::initial(7));
|
||||
assert_eq!(state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_and_get_status() {
|
||||
fn transitions_increment_revision_only_when_state_changes() {
|
||||
let state = test_state();
|
||||
state.set_status(WorkerStatus::Running);
|
||||
assert_eq!(state.get_status(), WorkerStatus::Running);
|
||||
state.set_status(WorkerStatus::Paused);
|
||||
assert_eq!(state.get_status(), WorkerStatus::Paused);
|
||||
let running = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running));
|
||||
let snapshot = state.transition(running.clone());
|
||||
assert_eq!(snapshot.revision, 1);
|
||||
assert_eq!(snapshot.state, running);
|
||||
assert_eq!(state.transition(running).revision, 1);
|
||||
|
||||
let paused = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused));
|
||||
let snapshot = state.transition(paused.clone());
|
||||
assert_eq!(snapshot.revision, 2);
|
||||
assert_eq!(snapshot.state, paused);
|
||||
assert_eq!(state.catalog_status(), WorkerStatus::Paused);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_json_contains_fields() {
|
||||
fn accepted_command_identity_advances_revision_and_detects_reuse_conflicts() {
|
||||
let state = test_state();
|
||||
let json = state.status_json();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed["state"], "idle");
|
||||
let envelope = WorkerCommandEnvelope {
|
||||
command_id: 9,
|
||||
expected_execution_generation: 7,
|
||||
expected_worker_state_revision: 0,
|
||||
};
|
||||
assert_eq!(
|
||||
state.admit_command(envelope, WorkerCommandKind::Pause, true),
|
||||
WorkerCommandAdmission::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
state.snapshot(),
|
||||
WorkerStateSnapshot {
|
||||
execution_generation: 7,
|
||||
revision: 1,
|
||||
last_command_id: 9,
|
||||
state: WorkerState::Idle,
|
||||
}
|
||||
);
|
||||
assert_eq!(state.command_result(9), Some(None));
|
||||
state.complete_command(
|
||||
9,
|
||||
WorkerCommandKind::Pause,
|
||||
WorkerCommandDisposition::Accepted,
|
||||
);
|
||||
assert_eq!(
|
||||
state.command_result(9),
|
||||
Some(Some(WorkerCommandDisposition::Accepted))
|
||||
);
|
||||
assert_eq!(
|
||||
state.admit_command(envelope, WorkerCommandKind::Pause, true),
|
||||
WorkerCommandAdmission::Retry
|
||||
);
|
||||
assert_eq!(
|
||||
state.admit_command(envelope, WorkerCommandKind::Cancel, true),
|
||||
WorkerCommandAdmission::Conflict
|
||||
);
|
||||
assert_eq!(state.snapshot().revision, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_json_contains_full_snapshot_and_catalog_projection() {
|
||||
let state = test_state();
|
||||
state.transition(WorkerState::Busy(WorkerBusyState::Maintenance(
|
||||
WorkerMaintenanceState::Compacting,
|
||||
)));
|
||||
let parsed: serde_json::Value = serde_json::from_str(&state.status_json()).unwrap();
|
||||
assert_eq!(parsed["state"], "running");
|
||||
assert_eq!(parsed["worker_state"]["execution_generation"], 7);
|
||||
assert_eq!(parsed["worker_state"]["revision"], 1);
|
||||
assert_eq!(parsed["worker_state"]["state"]["kind"], "busy");
|
||||
assert_eq!(parsed["worker_name"], "test-worker");
|
||||
assert!(parsed["segment_id"].is_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_json_reflects_changes() {
|
||||
let state = test_state();
|
||||
state.set_status(WorkerStatus::Running);
|
||||
let json = state.status_json();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed["state"], "running");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ mod tests {
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
},
|
||||
status: WorkerStatus::Idle,
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
}
|
||||
@@ -137,10 +137,16 @@ mod tests {
|
||||
],
|
||||
);
|
||||
|
||||
connect_and_send(&socket, &Method::Shutdown).await.unwrap();
|
||||
let method = Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::for_snapshot(
|
||||
1,
|
||||
&protocol::WorkerStateSnapshot::initial(1),
|
||||
),
|
||||
};
|
||||
connect_and_send(&socket, &method).await.unwrap();
|
||||
|
||||
let method = received.await.unwrap().expect("expected method");
|
||||
assert!(matches!(method, Method::Shutdown));
|
||||
assert!(matches!(method, Method::Shutdown { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+186
-20
@@ -4571,15 +4571,6 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn persist_and_send_compact_done(
|
||||
&mut self,
|
||||
lifecycle: CompactionLifecycle,
|
||||
) -> Result<(), WorkerError> {
|
||||
self.persist_compaction_lifecycle(&lifecycle)?;
|
||||
self.send_event(Event::CompactDone { lifecycle });
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn persist_and_send_compact_failed(
|
||||
&mut self,
|
||||
lifecycle: CompactionLifecycle,
|
||||
@@ -4724,7 +4715,97 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
Ok(rewrite_guard)
|
||||
}
|
||||
|
||||
/// Terminalize and clean up any compaction that was left active by the
|
||||
/// previous controller generation. This runs before the restored
|
||||
/// controller publishes its first Idle state.
|
||||
pub async fn recover_unfinished_compaction(&mut self) -> Result<(), WorkerError> {
|
||||
let (entries, _) = self.sink.subscribe_with_snapshot();
|
||||
let latest_payload = entries.iter().rev().find_map(|entry| match entry {
|
||||
LogEntry::Extension {
|
||||
domain, payload, ..
|
||||
} if domain == COMPACTION_EXTENSION_DOMAIN => Some(payload.clone()),
|
||||
_ => None,
|
||||
});
|
||||
let Some(payload) = latest_payload else {
|
||||
return Ok(());
|
||||
};
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct CompactionLifecycleWire {
|
||||
schema_version: u32,
|
||||
compaction_id: String,
|
||||
revision: u64,
|
||||
#[serde(default)]
|
||||
internal_worker: Option<protocol::InternalWorkerRef>,
|
||||
state: CompactionLifecycleState,
|
||||
started_at_ms: u64,
|
||||
#[serde(default)]
|
||||
ended_at_ms: Option<u64>,
|
||||
#[serde(default)]
|
||||
summary: Option<String>,
|
||||
#[serde(default)]
|
||||
error: Option<String>,
|
||||
#[serde(default)]
|
||||
new_segment_id: Option<String>,
|
||||
}
|
||||
let wire: CompactionLifecycleWire = serde_json::from_value(payload).map_err(|error| {
|
||||
WorkerError::InvalidState(format!("decode compaction lifecycle: {error}"))
|
||||
})?;
|
||||
if !matches!(wire.schema_version, 2 | 3) {
|
||||
return Err(WorkerError::InvalidState(format!(
|
||||
"unsupported compaction lifecycle schema version {}",
|
||||
wire.schema_version
|
||||
)));
|
||||
}
|
||||
let mut lifecycle = CompactionLifecycle {
|
||||
schema_version: wire.schema_version,
|
||||
compaction_id: wire.compaction_id,
|
||||
revision: wire.revision,
|
||||
internal_worker: wire.internal_worker,
|
||||
state: wire.state,
|
||||
started_at_ms: wire.started_at_ms,
|
||||
ended_at_ms: wire.ended_at_ms,
|
||||
summary: wire.summary,
|
||||
error: wire.error,
|
||||
new_segment_id: wire.new_segment_id,
|
||||
};
|
||||
match lifecycle.state {
|
||||
CompactionLifecycleState::Running => {
|
||||
lifecycle.schema_version = 3;
|
||||
lifecycle.revision = lifecycle.revision.saturating_add(1);
|
||||
lifecycle.state = CompactionLifecycleState::Interrupted;
|
||||
lifecycle.ended_at_ms = Some(segment_log::now_millis());
|
||||
lifecycle.error =
|
||||
Some("worker execution restarted before compaction completed".into());
|
||||
self.persist_compaction_lifecycle(&lifecycle)?;
|
||||
self.send_event(Event::CompactFailed {
|
||||
lifecycle: lifecycle.clone(),
|
||||
});
|
||||
self.release_compaction_service(&lifecycle).await;
|
||||
}
|
||||
CompactionLifecycleState::Interrupted => {
|
||||
self.release_compaction_service(&lifecycle).await;
|
||||
}
|
||||
CompactionLifecycleState::Done | CompactionLifecycleState::Failed => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn manual_compact(&mut self) -> Result<ManualCompactResult, WorkerError> {
|
||||
self.manual_compact_inner(None).await
|
||||
}
|
||||
|
||||
pub async fn manual_compact_with_cancel(
|
||||
&mut self,
|
||||
cancel: tokio::sync::watch::Receiver<bool>,
|
||||
) -> Result<ManualCompactResult, WorkerError> {
|
||||
self.manual_compact_inner(Some(cancel)).await
|
||||
}
|
||||
|
||||
async fn manual_compact_inner(
|
||||
&mut self,
|
||||
mut cancel: Option<tokio::sync::watch::Receiver<bool>>,
|
||||
) -> Result<ManualCompactResult, WorkerError> {
|
||||
if self.manifest.compaction.is_none() {
|
||||
let message =
|
||||
"manual compact is unavailable because [compaction] is not configured".to_string();
|
||||
@@ -4764,7 +4845,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
return Ok(ManualCompactResult::Skipped { message });
|
||||
}
|
||||
|
||||
match self.compact(retained).await {
|
||||
match self.compact_with_cancel(retained, cancel.take()).await {
|
||||
Ok(new_segment_id) => {
|
||||
info!(new_segment_id = %new_segment_id, "Manual compaction succeeded");
|
||||
if let Some(ref state) = state {
|
||||
@@ -4937,11 +5018,19 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
/// Runs one parent-owned observable compaction service and returns the new
|
||||
/// Segment ID. Lifecycle revisions are committed before they are broadcast.
|
||||
pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> {
|
||||
self.compact_with_cancel(retained_tokens, None).await
|
||||
}
|
||||
|
||||
async fn compact_with_cancel(
|
||||
&mut self,
|
||||
retained_tokens: u64,
|
||||
mut cancel: Option<tokio::sync::watch::Receiver<bool>>,
|
||||
) -> Result<SegmentId, WorkerError> {
|
||||
let _rewrite_guard = self
|
||||
.prepare_session_rewrite(SessionRewriteKind::Compact)
|
||||
.await?;
|
||||
let mut lifecycle = CompactionLifecycle {
|
||||
schema_version: 2,
|
||||
schema_version: 3,
|
||||
compaction_id: uuid::Uuid::now_v7().to_string(),
|
||||
revision: 1,
|
||||
internal_worker: None,
|
||||
@@ -4953,16 +5042,25 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
new_segment_id: None,
|
||||
};
|
||||
self.persist_and_send_compact_start(lifecycle.clone())?;
|
||||
match self.compact_impl(retained_tokens, &mut lifecycle).await {
|
||||
Ok((new_segment_id, summary)) => {
|
||||
lifecycle.revision = lifecycle.revision.saturating_add(1);
|
||||
lifecycle.state = CompactionLifecycleState::Done;
|
||||
lifecycle.ended_at_ms = Some(segment_log::now_millis());
|
||||
lifecycle.summary = Some(summary);
|
||||
lifecycle.new_segment_id = Some(new_segment_id.to_string());
|
||||
let terminal = self.persist_and_send_compact_done(lifecycle.clone());
|
||||
let outcome = if let Some(cancel) = cancel.as_mut() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
changed = cancel.changed() => {
|
||||
let _ = changed;
|
||||
Err(WorkerError::CompactCancelled)
|
||||
}
|
||||
result = self.compact_impl(retained_tokens, &mut lifecycle) => result,
|
||||
}
|
||||
} else {
|
||||
self.compact_impl(retained_tokens, &mut lifecycle).await
|
||||
};
|
||||
match outcome {
|
||||
Ok((new_segment_id, _summary)) => {
|
||||
debug_assert_eq!(lifecycle.state, CompactionLifecycleState::Done);
|
||||
self.send_event(Event::CompactDone {
|
||||
lifecycle: lifecycle.clone(),
|
||||
});
|
||||
self.release_compaction_service(&lifecycle).await;
|
||||
terminal?;
|
||||
Ok(new_segment_id)
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -5543,6 +5641,24 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
})?,
|
||||
});
|
||||
}
|
||||
// Commit the terminal lifecycle in the same atomic replacement-segment
|
||||
// creation as the rewritten history. Restore can therefore never see a
|
||||
// replacement segment without the Done fact for the compaction that
|
||||
// created it.
|
||||
lifecycle.revision = lifecycle.revision.saturating_add(1);
|
||||
lifecycle.state = CompactionLifecycleState::Done;
|
||||
lifecycle.ended_at_ms = Some(segment_log::now_millis());
|
||||
lifecycle.summary = Some(summary_text.clone());
|
||||
lifecycle.new_segment_id = Some(new_segment_id.to_string());
|
||||
initial_entries.push(LogEntry::Extension {
|
||||
ts: segment_log::now_millis(),
|
||||
domain: COMPACTION_EXTENSION_DOMAIN.to_string(),
|
||||
payload: serde_json::to_value(&*lifecycle).map_err(|error| {
|
||||
WorkerError::InvalidState(format!(
|
||||
"serialize terminal compaction lifecycle: {error}"
|
||||
))
|
||||
})?,
|
||||
});
|
||||
self.store
|
||||
.create_segment(old_loc.session_id, new_segment_id, &initial_entries)?;
|
||||
self.segment_state.set_location(SegmentLocation {
|
||||
@@ -10165,6 +10281,56 @@ mod build_summary_prompt_tests {
|
||||
assert_eq!(state.notification_receipts.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_terminalizes_running_compaction_before_idle_publication() {
|
||||
let (_dir, mut worker) = rewind_test_worker().await;
|
||||
let lifecycle = CompactionLifecycle {
|
||||
schema_version: 3,
|
||||
compaction_id: "compact-before-restart".into(),
|
||||
revision: 1,
|
||||
internal_worker: None,
|
||||
state: CompactionLifecycleState::Running,
|
||||
started_at_ms: segment_log::now_millis(),
|
||||
ended_at_ms: None,
|
||||
summary: None,
|
||||
error: None,
|
||||
new_segment_id: None,
|
||||
};
|
||||
worker.persist_compaction_lifecycle(&lifecycle).unwrap();
|
||||
|
||||
worker.recover_unfinished_compaction().await.unwrap();
|
||||
|
||||
let (entries, _) = worker.sink.subscribe_with_snapshot();
|
||||
let restored = entries.iter().rev().find_map(|entry| match entry {
|
||||
LogEntry::Extension {
|
||||
domain, payload, ..
|
||||
} if domain == COMPACTION_EXTENSION_DOMAIN => {
|
||||
serde_json::from_value::<CompactionLifecycle>(payload.clone()).ok()
|
||||
}
|
||||
_ => None,
|
||||
});
|
||||
let restored = restored.expect("terminal compaction lifecycle");
|
||||
assert_eq!(restored.state, CompactionLifecycleState::Interrupted);
|
||||
assert_eq!(restored.revision, 2);
|
||||
assert!(
|
||||
restored
|
||||
.error
|
||||
.as_deref()
|
||||
.is_some_and(|error| error.contains("restarted"))
|
||||
);
|
||||
|
||||
let mut future = lifecycle;
|
||||
future.schema_version = 4;
|
||||
future.compaction_id = "future-compaction".into();
|
||||
worker.persist_compaction_lifecycle(&future).unwrap();
|
||||
let error = worker.recover_unfinished_compaction().await.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("unsupported compaction lifecycle schema version 4")
|
||||
);
|
||||
}
|
||||
|
||||
fn minimal_manifest() -> WorkerManifest {
|
||||
let toml_str = r#"
|
||||
[worker]
|
||||
|
||||
@@ -72,6 +72,41 @@ impl LlmClient for MockClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BlockingCompactClient {
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl BlockingCompactClient {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmClient for BlockingCompactClient {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
async fn stream(
|
||||
&self,
|
||||
_request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if call == 0 {
|
||||
Ok(Box::pin(futures::stream::iter(
|
||||
single_text_events("seed").into_iter().map(Ok),
|
||||
)))
|
||||
} else {
|
||||
Ok(Box::pin(futures::stream::pending()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn single_text_events(text: &str) -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
@@ -156,10 +191,10 @@ target = "./"
|
||||
permission = "write"
|
||||
"#;
|
||||
|
||||
async fn make_worker_with_manifest(
|
||||
manifest_toml: &str,
|
||||
client: MockClient,
|
||||
) -> Worker<MockClient, TestStore> {
|
||||
async fn make_worker_with_manifest<C>(manifest_toml: &str, client: C) -> Worker<C, TestStore>
|
||||
where
|
||||
C: LlmClient + Clone + Send + Sync + 'static,
|
||||
{
|
||||
let manifest = worker::WorkerManifest::from_toml(manifest_toml).unwrap();
|
||||
|
||||
let store_tmp = tempfile::tempdir().unwrap();
|
||||
@@ -614,12 +649,144 @@ async fn pre_run_compact_failure_broadcasts_start_and_failed() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_compact_cancel_terminalizes_before_returning_idle() {
|
||||
let worker =
|
||||
make_worker_with_manifest(POST_RUN_MANIFEST_TOML, BlockingCompactClient::new()).await;
|
||||
let runtime_tmp = tempfile::tempdir().unwrap();
|
||||
let bash_output_dir = runtime_tmp.path().join("bash-output");
|
||||
let (handle, shutdown_receiver) =
|
||||
WorkerController::spawn(worker, runtime_tmp.path(), &bash_output_dir)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"seed history",
|
||||
))
|
||||
.await
|
||||
.expect("send seed run");
|
||||
loop {
|
||||
if matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||
.await
|
||||
.expect("timeout waiting for seed run")
|
||||
.expect("event"),
|
||||
Event::RunEnd {
|
||||
result: RunResult::Finished
|
||||
}
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let compact = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot());
|
||||
handle
|
||||
.send(Method::Compact { command: compact })
|
||||
.await
|
||||
.expect("send compact");
|
||||
loop {
|
||||
if matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||
.await
|
||||
.expect("timeout waiting for compact start")
|
||||
.expect("event"),
|
||||
Event::CompactStart { .. }
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let cancel = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot());
|
||||
handle
|
||||
.send(Method::Cancel { command: cancel })
|
||||
.await
|
||||
.expect("send compact cancel");
|
||||
let mut saw_interrupted = false;
|
||||
let mut saw_idle = false;
|
||||
while !(saw_interrupted && saw_idle) {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||
.await
|
||||
.expect("timeout waiting for compact cancellation")
|
||||
.expect("event")
|
||||
{
|
||||
Event::CompactFailed { lifecycle }
|
||||
if lifecycle.state == protocol::CompactionLifecycleState::Interrupted =>
|
||||
{
|
||||
saw_interrupted = true;
|
||||
}
|
||||
Event::WorkerState { snapshot }
|
||||
if snapshot.catalog_status() == protocol::WorkerStatus::Idle =>
|
||||
{
|
||||
assert!(
|
||||
saw_interrupted,
|
||||
"Idle must follow durable Interrupted evidence"
|
||||
);
|
||||
saw_idle = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let compact = protocol::WorkerCommandEnvelope::for_snapshot(3, &handle.shared_state.snapshot());
|
||||
handle
|
||||
.send(Method::Compact { command: compact })
|
||||
.await
|
||||
.expect("send second compact");
|
||||
loop {
|
||||
if matches!(
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||
.await
|
||||
.expect("timeout waiting for second compact start")
|
||||
.expect("event"),
|
||||
Event::CompactStart { .. }
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let shutdown =
|
||||
protocol::WorkerCommandEnvelope::for_snapshot(4, &handle.shared_state.snapshot());
|
||||
handle
|
||||
.send(Method::Shutdown { command: shutdown })
|
||||
.await
|
||||
.expect("send shutdown during compact");
|
||||
let mut interrupted_before_shutdown = false;
|
||||
loop {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||
.await
|
||||
.expect("timeout waiting for shutdown")
|
||||
.expect("event")
|
||||
{
|
||||
Event::CompactFailed { lifecycle }
|
||||
if lifecycle.state == protocol::CompactionLifecycleState::Interrupted =>
|
||||
{
|
||||
interrupted_before_shutdown = true;
|
||||
}
|
||||
Event::Shutdown => {
|
||||
assert!(
|
||||
interrupted_before_shutdown,
|
||||
"shutdown must await terminal compaction evidence"
|
||||
);
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), shutdown_receiver)
|
||||
.await
|
||||
.expect("controller shutdown timeout")
|
||||
.expect("shutdown confirmation");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn controller_compact_method_emits_start_and_done() {
|
||||
let client = MockClient::new(vec![
|
||||
text_events_with_usage("hi", 1000),
|
||||
write_summary_tool_use_events("manual-summary", "manual compact summary"),
|
||||
single_text_events("done"),
|
||||
single_text_events("follow-up"),
|
||||
]);
|
||||
let worker = make_worker_with_manifest(POST_RUN_MANIFEST_TOML, client).await;
|
||||
let runtime_tmp = tempfile::tempdir().unwrap();
|
||||
@@ -649,7 +816,11 @@ async fn controller_compact_method_emits_start_and_done() {
|
||||
}
|
||||
}
|
||||
|
||||
handle.send(Method::Compact).await.expect("send compact");
|
||||
let command = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot());
|
||||
handle
|
||||
.send(Method::Compact { command })
|
||||
.await
|
||||
.expect("send compact");
|
||||
let mut saw_start = false;
|
||||
loop {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||
@@ -670,5 +841,30 @@ async fn controller_compact_method_emits_start_and_done() {
|
||||
}
|
||||
|
||||
assert!(saw_start, "manual compact should emit CompactStart");
|
||||
let _ = handle.send(Method::Shutdown).await;
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"run after compact",
|
||||
))
|
||||
.await
|
||||
.expect("send follow-up run");
|
||||
loop {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||
.await
|
||||
.expect("timeout waiting for follow-up run")
|
||||
.expect("event")
|
||||
{
|
||||
Event::RunEnd {
|
||||
result: RunResult::Finished,
|
||||
} => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
handle.shared_state.catalog_status(),
|
||||
protocol::WorkerStatus::Idle,
|
||||
"successful manual compaction must release the execution fence"
|
||||
);
|
||||
let command = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot());
|
||||
let _ = handle.send(Method::Shutdown { command }).await;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use agen::Engine;
|
||||
@@ -25,6 +25,15 @@ use worker::{
|
||||
|
||||
type TestStore = CombinedStore<FsStore, FsWorkerStore>;
|
||||
|
||||
static NEXT_COMMAND_ID: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
fn worker_command(handle: &WorkerHandle) -> protocol::WorkerCommandEnvelope {
|
||||
protocol::WorkerCommandEnvelope::for_snapshot(
|
||||
NEXT_COMMAND_ID.fetch_add(1, Ordering::Relaxed),
|
||||
&handle.shared_state.snapshot(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Reconstruct a worker-history-like `Vec<Item>` from the live session
|
||||
/// log mirror held by the Worker's broadcast sink. Replaces the previous
|
||||
/// `WorkerSharedState.history()` test helper now that the mirror lives in
|
||||
@@ -313,7 +322,12 @@ async fn controller_grants_read_scope_for_exact_bash_output_directory() {
|
||||
}));
|
||||
assert!(!handle.runtime_dir.path().join("bash-output").exists());
|
||||
|
||||
handle.send(Method::Shutdown).await.unwrap();
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
shutdown_rx.await.unwrap();
|
||||
}
|
||||
|
||||
@@ -346,7 +360,12 @@ async fn shutdown_closes_bound_workdir_session() {
|
||||
WorkerController::spawn(worker, runtime_base.path(), &bash_output_dir)
|
||||
.await
|
||||
.unwrap();
|
||||
handle.send(Method::Shutdown).await.unwrap();
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
|
||||
.await
|
||||
.expect("controller should shut down")
|
||||
@@ -461,7 +480,12 @@ async fn controller_projects_workdir_command_events_and_snapshot_state() {
|
||||
!durable_history.contains("ready") && !durable_history.contains("done"),
|
||||
"operational command chunks must not be appended to Worker history: {durable_history}"
|
||||
);
|
||||
handle.send(Method::Shutdown).await.unwrap();
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -533,7 +557,12 @@ async fn controller_refreshes_command_snapshot_after_high_output_provider_lag()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(output.status, workdir::CommandStatus::Cancelled);
|
||||
handle.send(Method::Shutdown).await.unwrap();
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -575,13 +604,13 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
||||
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
if handle.shared_state.get_status() == status {
|
||||
if handle.shared_state.catalog_status() == status {
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"timed out waiting for status {status:?}; current={:?}",
|
||||
handle.shared_state.get_status()
|
||||
handle.shared_state.catalog_status()
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
@@ -1033,7 +1062,8 @@ async fn run_end_returns_to_idle_without_busy_status() {
|
||||
Ok(Event::RunEnd { result: protocol::RunResult::Finished }) => {
|
||||
saw_run_end = true;
|
||||
}
|
||||
Ok(Event::Status { status: WorkerStatus::Idle }) if saw_run_end => {
|
||||
Ok(Event::WorkerState { snapshot })
|
||||
if saw_run_end && snapshot.catalog_status() == WorkerStatus::Idle => {
|
||||
saw_idle_status = true;
|
||||
break;
|
||||
}
|
||||
@@ -1050,7 +1080,7 @@ async fn run_end_returns_to_idle_without_busy_status() {
|
||||
saw_idle_status,
|
||||
"expected idle status immediately after RunEnd"
|
||||
);
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1128,9 +1158,7 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
|
||||
loop {
|
||||
if matches!(
|
||||
events.recv().await,
|
||||
Ok(Event::Status {
|
||||
status: WorkerStatus::Running,
|
||||
})
|
||||
Ok(Event::WorkerState { snapshot }) if snapshot.catalog_status() == WorkerStatus::Running
|
||||
) {
|
||||
break;
|
||||
}
|
||||
@@ -1205,8 +1233,8 @@ async fn attach_snapshot_includes_current_status() {
|
||||
loop {
|
||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||
match event {
|
||||
Event::Snapshot { status, .. } => {
|
||||
assert_eq!(status, WorkerStatus::Running);
|
||||
Event::Snapshot { state, .. } => {
|
||||
assert_eq!(state.catalog_status(), WorkerStatus::Running);
|
||||
return;
|
||||
}
|
||||
Event::Alert(_) => continue,
|
||||
@@ -1221,7 +1249,7 @@ async fn shared_state_starts_idle() {
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1241,7 +1269,7 @@ async fn run_updates_shared_state_to_idle_after_completion() {
|
||||
// Wait for the run to complete
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1364,7 +1392,12 @@ async fn submit_while_running_is_durably_queued() {
|
||||
assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued));
|
||||
let pending_snapshot = pending_snapshot.expect("pending snapshot");
|
||||
assert_eq!(pending_snapshot.submissions.len(), 1);
|
||||
handle.send(Method::Pause).await.unwrap();
|
||||
handle
|
||||
.send(Method::Pause {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Paused).await;
|
||||
handle
|
||||
.send(Method::ContinuePending {
|
||||
@@ -1386,17 +1419,22 @@ async fn submit_while_running_is_durably_queued() {
|
||||
.await
|
||||
.expect("paused ContinuePending rejection");
|
||||
assert!(rejection.contains("Resume or Cancel"));
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resume_without_pause_returns_error() {
|
||||
async fn resume_without_pause_returns_invalid_state_acknowledgement() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::Resume).await.unwrap();
|
||||
handle
|
||||
.send(Method::Resume {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut saw_not_paused = false;
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||
@@ -1404,7 +1442,10 @@ async fn resume_without_pause_returns_error() {
|
||||
tokio::select! {
|
||||
event = rx.recv() => {
|
||||
match event {
|
||||
Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotPaused => {
|
||||
Ok(Event::CommandAcknowledged { acknowledgement })
|
||||
if acknowledgement.command == protocol::WorkerCommandKind::Resume
|
||||
&& acknowledgement.disposition
|
||||
== protocol::WorkerCommandDisposition::InvalidState => {
|
||||
saw_not_paused = true;
|
||||
break;
|
||||
}
|
||||
@@ -1416,17 +1457,22 @@ async fn resume_without_pause_returns_error() {
|
||||
}
|
||||
}
|
||||
|
||||
assert!(saw_not_paused, "should see not_paused error");
|
||||
assert!(saw_not_paused, "should see invalid-state acknowledgement");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_without_run_returns_error() {
|
||||
async fn cancel_without_run_returns_invalid_state_acknowledgement() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
handle.send(Method::Cancel).await.unwrap();
|
||||
handle
|
||||
.send(Method::Cancel {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut saw_not_running = false;
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||
@@ -1434,7 +1480,10 @@ async fn cancel_without_run_returns_error() {
|
||||
tokio::select! {
|
||||
event = rx.recv() => {
|
||||
match event {
|
||||
Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotRunning => {
|
||||
Ok(Event::CommandAcknowledged { acknowledgement })
|
||||
if acknowledgement.command == protocol::WorkerCommandKind::Cancel
|
||||
&& acknowledgement.disposition
|
||||
== protocol::WorkerCommandDisposition::InvalidState => {
|
||||
saw_not_running = true;
|
||||
break;
|
||||
}
|
||||
@@ -1446,7 +1495,7 @@ async fn cancel_without_run_returns_error() {
|
||||
}
|
||||
}
|
||||
|
||||
assert!(saw_not_running, "should see not_running error");
|
||||
assert!(saw_not_running, "should see invalid-state acknowledgement");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1822,7 +1871,7 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
|
||||
}
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
assert!(
|
||||
client_for_assert.captured_requests().is_empty(),
|
||||
"weak Notify must not stage RunForNotification while idle"
|
||||
@@ -1919,7 +1968,7 @@ async fn worker_event_turn_ended_while_idle_auto_starts_turn_and_injects_system_
|
||||
saw_worker_event_in_mirror,
|
||||
"Method::WorkerEvent should commit a SystemItem::WorkerEvent entry"
|
||||
);
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
|
||||
let requests = client_for_assert.captured_requests();
|
||||
assert_eq!(
|
||||
@@ -1982,7 +2031,7 @@ async fn worker_event_scope_sub_delegated_while_idle_stays_control_plane_only()
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
|
||||
assert_eq!(
|
||||
handle.shared_state.get_status(),
|
||||
handle.shared_state.catalog_status(),
|
||||
WorkerStatus::Idle,
|
||||
"control-plane ScopeSubDelegated must not auto-start the parent LLM"
|
||||
);
|
||||
@@ -2085,7 +2134,12 @@ async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() {
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
handle.send(Method::Cancel).await.unwrap();
|
||||
handle
|
||||
.send(Method::Cancel {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
|
||||
let mut rx = handle.subscribe();
|
||||
@@ -2482,7 +2536,12 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
"text_delta should arrive before pause"
|
||||
);
|
||||
|
||||
handle.send(Method::Pause).await.unwrap();
|
||||
handle
|
||||
.send(Method::Pause {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The controller emits RunEnd { Paused } when the
|
||||
// EngineError::Cancelled is translated under pause_requested.
|
||||
@@ -2498,9 +2557,14 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
);
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
|
||||
|
||||
handle.send(Method::Resume).await.unwrap();
|
||||
handle
|
||||
.send(Method::Resume {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
@@ -2514,7 +2578,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
);
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
|
||||
// History consistency: exactly [user "hello", assistant
|
||||
// "resumed output"]. No artifacts from the aborted stream
|
||||
@@ -2614,7 +2678,12 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
|
||||
"tool_call_done should arrive before pause"
|
||||
);
|
||||
|
||||
handle.send(Method::Pause).await.unwrap();
|
||||
handle
|
||||
.send(Method::Pause {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
e,
|
||||
@@ -2626,7 +2695,7 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
|
||||
"expected RunEnd::Paused"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
|
||||
|
||||
// New user input while Paused → `Worker::run` observes
|
||||
// `last_run_interrupted` and runs its interrupt-prep step, which
|
||||
@@ -2785,7 +2854,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
||||
"tool_call_done should arrive before pause"
|
||||
);
|
||||
|
||||
handle.send(Method::Pause).await.unwrap();
|
||||
handle
|
||||
.send(Method::Pause {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
e,
|
||||
@@ -2798,7 +2872,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
||||
);
|
||||
wait_for_status(&handle, WorkerStatus::Paused).await;
|
||||
|
||||
handle.send(Method::Cancel).await.unwrap();
|
||||
handle
|
||||
.send(Method::Cancel {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
let (entries_after_cancel, _rx_after_cancel) = handle.sink.subscribe_with_snapshot();
|
||||
assert!(
|
||||
@@ -2824,17 +2903,22 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
||||
"paused cancel must not resume or start another LLM request"
|
||||
);
|
||||
|
||||
handle.send(Method::Resume).await.unwrap();
|
||||
handle
|
||||
.send(Method::Resume {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
e,
|
||||
Event::Error {
|
||||
code: worker::ErrorCode::NotPaused,
|
||||
..
|
||||
}
|
||||
Event::CommandAcknowledged { acknowledgement }
|
||||
if acknowledgement.command == protocol::WorkerCommandKind::Resume
|
||||
&& acknowledgement.disposition
|
||||
== protocol::WorkerCommandDisposition::InvalidState
|
||||
))
|
||||
.await,
|
||||
"resume after paused cancel should be rejected as not paused"
|
||||
"resume after paused cancel should receive invalid-state acknowledgement"
|
||||
);
|
||||
assert_eq!(
|
||||
client_for_assert.captured_requests().len(),
|
||||
@@ -2943,7 +3027,12 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() {
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
handle.send(Method::Cancel).await.unwrap();
|
||||
handle
|
||||
.send(Method::Cancel {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
@@ -2981,7 +3070,12 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() {
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
handle.send(Method::Pause).await.unwrap();
|
||||
handle
|
||||
.send(Method::Pause {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
@@ -3038,7 +3132,12 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() {
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
handle.send(Method::Cancel).await.unwrap();
|
||||
handle
|
||||
.send(Method::Cancel {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
e,
|
||||
@@ -3095,7 +3194,12 @@ async fn pause_after_assistant_token_does_not_rollback() {
|
||||
.await,
|
||||
"assistant token should be visible before pause"
|
||||
);
|
||||
handle.send(Method::Pause).await.unwrap();
|
||||
handle
|
||||
.send(Method::Pause {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||
|
||||
@@ -1904,7 +1904,12 @@ pub struct WorkerSummary {
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
pub workspace: WorkerWorkspaceSummary,
|
||||
/// Runtime catalog lifecycle compatibility state. Live foreground state, when
|
||||
/// available, is carried separately in `worker_state`.
|
||||
pub state: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[cfg_attr(feature = "typescript", ts(optional))]
|
||||
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||
pub last_seen_at: Option<String>,
|
||||
#[serde(default)]
|
||||
pub pinned: bool,
|
||||
|
||||
@@ -36,8 +36,6 @@ use worker_runtime::config_bundle::{
|
||||
ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
|
||||
};
|
||||
use worker_runtime::error::RuntimeError as EmbeddedRuntimeError;
|
||||
#[cfg(test)]
|
||||
use worker_runtime::execution::WorkerExecutionRunState;
|
||||
use worker_runtime::fs_store::FsRuntimeStoreOptions;
|
||||
use worker_runtime::http_server::{
|
||||
RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER,
|
||||
@@ -245,7 +243,10 @@ pub struct WorkerSummary {
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
pub workspace: WorkerWorkspaceSummary,
|
||||
/// Runtime catalog lifecycle compatibility state.
|
||||
pub state: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||
pub last_seen_at: Option<String>,
|
||||
#[serde(default)]
|
||||
pub pinned: bool,
|
||||
@@ -337,6 +338,7 @@ pub(crate) fn workspace_worker_summary(
|
||||
workspace_id: summary.workspace.workspace_id,
|
||||
},
|
||||
state: summary.state,
|
||||
worker_state: summary.worker_state,
|
||||
last_seen_at: summary.last_seen_at,
|
||||
pinned: summary.pinned,
|
||||
retention_state: summary.retention_state,
|
||||
@@ -2000,6 +2002,7 @@ impl EmbeddedWorkerRuntime {
|
||||
workspace_id: summary.workspace_id.clone(),
|
||||
},
|
||||
state: embedded_worker_status_label(summary.status).to_string(),
|
||||
worker_state: summary.worker_state.clone(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
@@ -2039,6 +2042,7 @@ impl EmbeddedWorkerRuntime {
|
||||
workspace_id: detail.workspace_id.clone(),
|
||||
},
|
||||
state: embedded_worker_status_label(detail.status).to_string(),
|
||||
worker_state: detail.worker_state.clone(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
@@ -3342,6 +3346,7 @@ impl RemoteWorkerRuntime {
|
||||
workspace_id: summary.workspace_id.clone(),
|
||||
},
|
||||
state: embedded_worker_status_label(summary.status).to_string(),
|
||||
worker_state: summary.worker_state.clone(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
@@ -3385,6 +3390,7 @@ impl RemoteWorkerRuntime {
|
||||
workspace_id: detail.workspace_id.clone(),
|
||||
},
|
||||
state: embedded_worker_status_label(detail.status).to_string(),
|
||||
worker_state: detail.worker_state.clone(),
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
@@ -4732,6 +4738,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> WorkerSummary {
|
||||
workspace_id: None,
|
||||
},
|
||||
state: "unsupported".to_string(),
|
||||
worker_state: None,
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
@@ -5170,7 +5177,6 @@ mod tests {
|
||||
request.worker_ref,
|
||||
self.backend_id(),
|
||||
),
|
||||
run_state: WorkerExecutionRunState::Idle,
|
||||
working_directory: request
|
||||
.working_directory
|
||||
.as_ref()
|
||||
@@ -5199,8 +5205,8 @@ mod tests {
|
||||
let content = input.content;
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
let _ = context.publish_protocol_event(protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Running,
|
||||
let _ = context.publish_protocol_event(protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStatus::Running.into(),
|
||||
});
|
||||
let _ = context.publish_protocol_event(protocol::Event::TextDone {
|
||||
text: format!("echo: {content}"),
|
||||
@@ -5208,14 +5214,13 @@ mod tests {
|
||||
let _ = context.publish_protocol_event(protocol::Event::RunEnd {
|
||||
result: protocol::RunResult::Finished,
|
||||
});
|
||||
let _ = context.publish_protocol_event(protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Idle,
|
||||
let _ = context.publish_protocol_event(protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStatus::Idle.into(),
|
||||
});
|
||||
});
|
||||
if let Some(submission_request_id) = submission_request_id {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
submission_request_id,
|
||||
uuid::Uuid::now_v7().to_string(),
|
||||
protocol::SubmissionDisposition::Started,
|
||||
@@ -5223,7 +5228,6 @@ mod tests {
|
||||
} else {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -5256,6 +5260,7 @@ mod tests {
|
||||
workspace_id: None,
|
||||
},
|
||||
state: "available".to_string(),
|
||||
worker_state: None,
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "transient".to_string(),
|
||||
|
||||
@@ -6,7 +6,7 @@ use worker_runtime::catalog::{
|
||||
};
|
||||
use worker_runtime::execution::{
|
||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult,
|
||||
WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||
};
|
||||
use worker_runtime::identity::WorkerId;
|
||||
use worker_runtime::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary};
|
||||
@@ -22,7 +22,6 @@ impl WorkerExecutionBackend for TestExecutionBackend {
|
||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::connected(
|
||||
WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||
WorkerExecutionRunState::Idle,
|
||||
None,
|
||||
)
|
||||
}
|
||||
@@ -35,24 +34,17 @@ impl WorkerExecutionBackend for TestExecutionBackend {
|
||||
if let Some(submission_request_id) = input.submission_request_id {
|
||||
WorkerExecutionResult::accepted_submission(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
submission_request_id,
|
||||
uuid::Uuid::now_v7().to_string(),
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
} else {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Input,
|
||||
WorkerExecutionRunState::Busy,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||
WorkerExecutionResult::accepted(
|
||||
WorkerExecutionOperation::Stop,
|
||||
WorkerExecutionRunState::Stopped,
|
||||
)
|
||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,8 +191,8 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
&worker.worker_ref,
|
||||
protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Running,
|
||||
protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStatus::Running.into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -210,7 +202,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
||||
BrokerSubscriptionEvent::Event {
|
||||
payload: SubscriptionEventPayload::WorkerUpserted { ref worker },
|
||||
..
|
||||
} if worker.state == SubscriptionWorkerState::Running
|
||||
} if worker.state == SubscriptionWorkerState::Idle
|
||||
&& matches!(
|
||||
worker.worker_state,
|
||||
Some(protocol::WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running
|
||||
)),
|
||||
..
|
||||
})
|
||||
)
|
||||
));
|
||||
}
|
||||
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
||||
@@ -220,7 +221,18 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
||||
assert!(matches!(
|
||||
snapshot,
|
||||
SubscriptionSnapshot::Workers { workers }
|
||||
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
||||
if workers.iter().any(|worker| {
|
||||
worker.state == SubscriptionWorkerState::Idle
|
||||
&& matches!(
|
||||
worker.worker_state,
|
||||
Some(protocol::WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Busy(
|
||||
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
|
||||
),
|
||||
..
|
||||
})
|
||||
)
|
||||
})
|
||||
));
|
||||
drop(late);
|
||||
|
||||
@@ -339,14 +351,24 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
|
||||
runtime
|
||||
.observe_worker_event(
|
||||
&worker.worker_ref,
|
||||
protocol::Event::Status {
|
||||
status: protocol::WorkerStatus::Running,
|
||||
protocol::Event::WorkerState {
|
||||
snapshot: protocol::WorkerStatus::Running.into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(next_event(&mut subscription).await,
|
||||
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
|
||||
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running));
|
||||
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime")
|
||||
&& worker.state == SubscriptionWorkerState::Idle
|
||||
&& matches!(
|
||||
worker.worker_state,
|
||||
Some(protocol::WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running
|
||||
)),
|
||||
..
|
||||
})
|
||||
)));
|
||||
let mut late = broker
|
||||
.subscribe(
|
||||
"embedded-worker-runtime",
|
||||
@@ -359,7 +381,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
|
||||
assert!(matches!(
|
||||
snapshot,
|
||||
SubscriptionSnapshot::Workers { workers }
|
||||
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
||||
if workers.iter().any(|worker| {
|
||||
worker.state == SubscriptionWorkerState::Idle
|
||||
&& matches!(
|
||||
worker.worker_state,
|
||||
Some(protocol::WorkerStateSnapshot {
|
||||
state: protocol::WorkerState::Busy(
|
||||
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
|
||||
),
|
||||
..
|
||||
})
|
||||
)
|
||||
})
|
||||
));
|
||||
|
||||
runtime
|
||||
|
||||
@@ -15766,6 +15766,7 @@ fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> WorkerSummary
|
||||
singleton_key: None,
|
||||
tags: Vec::new(),
|
||||
state: "missing".to_string(),
|
||||
worker_state: None,
|
||||
last_seen_at: Some(record.updated_at.clone()),
|
||||
pinned: record.retention_state == "pinned",
|
||||
retention_state: record.retention_state.clone(),
|
||||
@@ -19422,7 +19423,6 @@ mod tests {
|
||||
request.worker_ref,
|
||||
self.backend_id(),
|
||||
),
|
||||
run_state: worker_runtime::execution::WorkerExecutionRunState::Idle,
|
||||
working_directory,
|
||||
}
|
||||
}
|
||||
@@ -19438,17 +19438,43 @@ mod tests {
|
||||
.push((handle.worker_ref().clone(), method));
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||
worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod,
|
||||
worker_runtime::execution::WorkerExecutionRunState::Idle,
|
||||
)
|
||||
}
|
||||
|
||||
fn worker_snapshot(
|
||||
&self,
|
||||
handle: &worker_runtime::execution::WorkerExecutionHandle,
|
||||
) -> Option<protocol::Event> {
|
||||
Some(protocol::Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: Vec::new(),
|
||||
},
|
||||
greeting: protocol::Greeting {
|
||||
worker_name: handle.worker_ref().worker_id.to_string(),
|
||||
cwd: String::new(),
|
||||
provider: "deterministic-workspace-server-test".to_string(),
|
||||
model: "deterministic-workspace-server-test".to_string(),
|
||||
scope_summary: "test execution snapshot".to_string(),
|
||||
tools: Vec::new(),
|
||||
context_window: 0,
|
||||
context_tokens: 0,
|
||||
},
|
||||
state: protocol::WorkerStateSnapshot::initial(1),
|
||||
in_flight: protocol::InFlightSnapshot {
|
||||
blocks: Vec::new(),
|
||||
commands: Vec::new(),
|
||||
},
|
||||
internal_workers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn stop_worker(
|
||||
&self,
|
||||
_handle: &worker_runtime::execution::WorkerExecutionHandle,
|
||||
) -> worker_runtime::execution::WorkerExecutionResult {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Stop,
|
||||
worker_runtime::execution::WorkerExecutionRunState::Stopped,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -19458,7 +19484,6 @@ mod tests {
|
||||
) -> worker_runtime::execution::WorkerExecutionResult {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Cancel,
|
||||
worker_runtime::execution::WorkerExecutionRunState::Stopped,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -19495,16 +19520,16 @@ mod tests {
|
||||
if let Some(submission_request_id) = submission_request_id {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||
worker_runtime::execution::WorkerExecutionRunState::Idle,
|
||||
submission_request_id,
|
||||
uuid::Uuid::now_v7().to_string(),
|
||||
protocol::SubmissionDisposition::Started,
|
||||
)
|
||||
.with_worker_state(protocol::WorkerStateSnapshot::initial(1))
|
||||
} else {
|
||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||
worker_runtime::execution::WorkerExecutionRunState::Idle,
|
||||
)
|
||||
.with_worker_state(protocol::WorkerStateSnapshot::initial(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25570,6 +25595,7 @@ mod tests {
|
||||
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
||||
},
|
||||
state: "idle".to_string(),
|
||||
worker_state: None,
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "normal".to_string(),
|
||||
@@ -25666,6 +25692,7 @@ mod tests {
|
||||
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
||||
},
|
||||
state: "idle".to_string(),
|
||||
worker_state: None,
|
||||
last_seen_at: None,
|
||||
pinned: false,
|
||||
retention_state: "normal".to_string(),
|
||||
@@ -28556,7 +28583,13 @@ mod tests {
|
||||
protocol::subscription::SubscriptionFramePayload::WorkerProtocol(
|
||||
protocol::subscription::SubscriptionWorkerProtocolMethod {
|
||||
subscription_id: second_protocol_subscription_id,
|
||||
method: protocol::Method::Resume,
|
||||
method: protocol::Method::Resume {
|
||||
command: protocol::WorkerCommandEnvelope {
|
||||
command_id: 1,
|
||||
expected_execution_generation: 1,
|
||||
expected_worker_state_revision: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
@@ -28573,7 +28606,7 @@ mod tests {
|
||||
.iter()
|
||||
.any(|(worker_ref, method)| {
|
||||
worker_ref.worker_id.to_string() == worker_id
|
||||
&& matches!(method, protocol::Method::Resume)
|
||||
&& matches!(method, protocol::Method::Resume { .. })
|
||||
})
|
||||
{
|
||||
break;
|
||||
@@ -28586,7 +28619,7 @@ mod tests {
|
||||
let protocol_methods = execution_backend.protocol_methods();
|
||||
assert!(protocol_methods.iter().any(|(worker_ref, method)| {
|
||||
worker_ref.worker_id.to_string() == worker_id
|
||||
&& matches!(method, protocol::Method::Resume)
|
||||
&& matches!(method, protocol::Method::Resume { .. })
|
||||
}));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
@@ -10,6 +10,37 @@ export type CompletionKind = "file";
|
||||
|
||||
export type WorkerStatus = "idle" | "running" | "paused" | "stopped";
|
||||
|
||||
export type WorkerCommandEnvelope = {
|
||||
/**
|
||||
* Caller-owned sequence. A controller accepts command ids in strictly
|
||||
* increasing order for one execution generation.
|
||||
*/
|
||||
command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, };
|
||||
|
||||
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
|
||||
|
||||
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
|
||||
|
||||
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
|
||||
/**
|
||||
* The complete authoritative state observed after command admission.
|
||||
*/
|
||||
state: WorkerStateSnapshot, };
|
||||
|
||||
export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
|
||||
|
||||
export type WorkerMaintenanceState = "compacting";
|
||||
|
||||
export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "kind": "maintenance", "state": WorkerMaintenanceState };
|
||||
|
||||
export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState };
|
||||
|
||||
export type WorkerStateSnapshot = { execution_generation: number, revision: number,
|
||||
/**
|
||||
* Highest lifecycle command id observed by this controller generation.
|
||||
*/
|
||||
last_command_id: number, state: WorkerState, };
|
||||
|
||||
export type TurnResult = "finished" | "paused";
|
||||
|
||||
export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup";
|
||||
@@ -202,7 +233,16 @@ resource_key?: string | null,
|
||||
/**
|
||||
* Producer-owned monotonic revision for this Worker subject.
|
||||
*/
|
||||
subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
||||
subject_revision: number,
|
||||
/**
|
||||
* Latest revisioned foreground state observed from the Worker. This remains
|
||||
* absent until an authoritative Worker snapshot/event has been applied.
|
||||
*/
|
||||
worker_state?: WorkerStateSnapshot | null,
|
||||
/**
|
||||
* Runtime catalog lifecycle compatibility projection; not foreground-state authority.
|
||||
*/
|
||||
state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
||||
/**
|
||||
* Workspace-facing Repository key. Runtime producers leave this unset and
|
||||
* Workspace Server projections replace `repository_id` with this field.
|
||||
@@ -231,7 +271,7 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip
|
||||
|
||||
export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod });
|
||||
|
||||
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
||||
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
||||
|
||||
export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
|
||||
/**
|
||||
@@ -247,7 +287,12 @@ summary: string,
|
||||
* Full tool output. Absent when the tool chose to return
|
||||
* summary-only, or when the result was pruned.
|
||||
*/
|
||||
output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, status: WorkerStatus,
|
||||
output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting,
|
||||
/**
|
||||
* Full revisioned live execution state. `Stopped` remains Runtime
|
||||
* catalog authority and is deliberately not represented here.
|
||||
*/
|
||||
state: WorkerStateSnapshot,
|
||||
/**
|
||||
* Unfinished model output that has already streamed in the current
|
||||
* run but is not yet represented by committed snapshot entries.
|
||||
@@ -257,4 +302,4 @@ in_flight?: InFlightSnapshot,
|
||||
* Parent-owned Internal Worker sessions visible to this client.
|
||||
* Service-private Internal Workers are deliberately excluded.
|
||||
*/
|
||||
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" };
|
||||
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" };
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { Event } from "$lib/generated/protocol";
|
||||
import type { Event, WorkerStateSnapshot, WorkerStatus } from "$lib/generated/protocol";
|
||||
import {
|
||||
type ConsoleEventInput,
|
||||
type ConsoleLine,
|
||||
@@ -19,6 +19,23 @@ declare const Deno: {
|
||||
test(name: string, fn: () => void): void;
|
||||
};
|
||||
|
||||
function workerState(status: WorkerStatus): WorkerStateSnapshot {
|
||||
return {
|
||||
execution_generation: 1,
|
||||
revision: status === "idle" ? 0 : 1,
|
||||
last_command_id: 0,
|
||||
state: status === "idle"
|
||||
? { kind: "idle" }
|
||||
: {
|
||||
kind: "busy",
|
||||
state: {
|
||||
kind: "run",
|
||||
state: status === "paused" ? "paused" : "running",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
@@ -131,7 +148,7 @@ function snapshotEvent(cwd: string, entries: unknown[] = []): Event {
|
||||
context_window: 100,
|
||||
context_tokens: 20,
|
||||
},
|
||||
status: "idle",
|
||||
state: workerState("idle"),
|
||||
in_flight: { blocks: [] },
|
||||
},
|
||||
};
|
||||
@@ -201,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", ()
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
|
||||
const projector = createConsoleProjector();
|
||||
const running: WorkerStateSnapshot = {
|
||||
execution_generation: 4,
|
||||
revision: 3,
|
||||
last_command_id: 2,
|
||||
state: { kind: "busy", state: { kind: "run", state: "running" } },
|
||||
};
|
||||
const paused: WorkerStateSnapshot = {
|
||||
...running,
|
||||
revision: 4,
|
||||
last_command_id: 3,
|
||||
state: { kind: "busy", state: { kind: "run", state: "paused" } },
|
||||
};
|
||||
let projection = projector.append([
|
||||
{
|
||||
eventId: "running",
|
||||
event: { event: "worker_state", data: { snapshot: running } },
|
||||
},
|
||||
{
|
||||
eventId: "stale",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
|
||||
},
|
||||
},
|
||||
{
|
||||
eventId: "pause-ack",
|
||||
event: {
|
||||
event: "command_acknowledged",
|
||||
data: {
|
||||
acknowledgement: {
|
||||
command_id: 3,
|
||||
command: "pause",
|
||||
disposition: "accepted",
|
||||
state: paused,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
assertEquals(projection.workerState, paused);
|
||||
assertEquals(projection.status, "paused");
|
||||
|
||||
projection = projector.append([{
|
||||
eventId: "conflict",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...paused, state: { kind: "idle" } } },
|
||||
},
|
||||
}]);
|
||||
assertEquals(projection.workerState, paused);
|
||||
assert(
|
||||
projection.lines.some((line) =>
|
||||
line.eventId === "conflict:worker-state-conflict" && line.error
|
||||
),
|
||||
"conflicting equal-version snapshots must fail closed",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("snapshot replaces a live error with one durable run_errored row", () => {
|
||||
const projector = createConsoleProjector();
|
||||
let projection = projector.append([
|
||||
@@ -213,7 +290,7 @@ Deno.test("snapshot replaces a live error with one durable run_errored row", ()
|
||||
},
|
||||
{
|
||||
eventId: "idle-after-error",
|
||||
event: { event: "status", data: { status: "idle" } } satisfies Event,
|
||||
event: { event: "worker_state", data: { snapshot: workerState("idle") } } satisfies Event,
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -653,7 +730,7 @@ Deno.test("projectConsole streams distinct Bash stdout and stderr through termin
|
||||
Deno.test("snapshot restores bounded in-flight Bash command output", () => {
|
||||
const snapshot = snapshotEvent("/repo");
|
||||
if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected");
|
||||
snapshot.data.status = "running";
|
||||
snapshot.data.state = workerState("running");
|
||||
snapshot.data.in_flight = {
|
||||
blocks: [{
|
||||
kind: "tool_call",
|
||||
@@ -1403,7 +1480,7 @@ Deno.test("projectConsole hides lifecycle events and renders system items", () =
|
||||
const projection = projectConsole([
|
||||
{
|
||||
eventId: "30",
|
||||
event: { event: "status", data: { status: "running" } } satisfies Event,
|
||||
event: { event: "worker_state", data: { snapshot: workerState("running") } } satisfies Event,
|
||||
},
|
||||
{
|
||||
eventId: "31",
|
||||
@@ -1527,7 +1604,7 @@ Deno.test("projectConsole renders snapshot entries and in-flight output", () =>
|
||||
context_window: 100,
|
||||
context_tokens: 20,
|
||||
},
|
||||
status: "running",
|
||||
state: workerState("running"),
|
||||
in_flight: {
|
||||
blocks: [
|
||||
{ kind: "text", text: "partial" },
|
||||
@@ -1578,7 +1655,7 @@ Deno.test("projectConsole restores system items from snapshot entries", () => {
|
||||
context_window: 100,
|
||||
context_tokens: 20,
|
||||
},
|
||||
status: "idle",
|
||||
state: workerState("idle"),
|
||||
},
|
||||
} satisfies Event,
|
||||
}]);
|
||||
@@ -1922,7 +1999,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
|
||||
kind: "sub_worker",
|
||||
},
|
||||
revision: 1,
|
||||
event: { event: "status", data: { status: "running" } },
|
||||
event: { event: "worker_state", data: { snapshot: workerState("running") } },
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1941,7 +2018,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
|
||||
kind: "sub_worker",
|
||||
},
|
||||
revision: 1,
|
||||
event: { event: "status", data: { status: "idle" } },
|
||||
event: { event: "worker_state", data: { snapshot: workerState("idle") } },
|
||||
},
|
||||
},
|
||||
}]);
|
||||
@@ -2033,7 +2110,7 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections"
|
||||
kind: "sub_worker",
|
||||
},
|
||||
revision: 1,
|
||||
event: { event: "status", data: { status: "running" } },
|
||||
event: { event: "worker_state", data: { snapshot: workerState("running") } },
|
||||
},
|
||||
},
|
||||
}]);
|
||||
|
||||
@@ -10,6 +10,9 @@ import type {
|
||||
InternalWorkerRef,
|
||||
InternalWorkerSnapshot,
|
||||
Segment,
|
||||
WorkerState,
|
||||
WorkerStateSnapshot,
|
||||
WorkerStatus,
|
||||
} from "$lib/generated/protocol";
|
||||
import { stringify as stringifyYaml } from "yaml";
|
||||
import { workspaceRoute } from "$lib/workspace/api/http";
|
||||
@@ -169,6 +172,7 @@ export type ConsoleProjection = {
|
||||
tasks: ConsoleTask[];
|
||||
taskNextId: number;
|
||||
status: string | null;
|
||||
workerState: WorkerStateSnapshot | null;
|
||||
usage: string | null;
|
||||
runActivity: RunActivityStats;
|
||||
cwd: string | null;
|
||||
@@ -251,12 +255,22 @@ export function isConsoleProjectionEvent(event: ProtocolEvent): boolean {
|
||||
return event.event !== "completions";
|
||||
}
|
||||
|
||||
function workerStatusFromState(snapshot: WorkerStateSnapshot): WorkerStatus {
|
||||
if (snapshot.state.kind === "idle") return "idle";
|
||||
if (
|
||||
snapshot.state.state.kind === "run" &&
|
||||
snapshot.state.state.state === "paused"
|
||||
) return "paused";
|
||||
return "running";
|
||||
}
|
||||
|
||||
export function emptyConsoleProjection(): ConsoleProjection {
|
||||
return {
|
||||
lines: [],
|
||||
tasks: [],
|
||||
taskNextId: 1,
|
||||
status: null,
|
||||
workerState: null,
|
||||
usage: null,
|
||||
runActivity: emptyRunActivityStats(),
|
||||
cwd: null,
|
||||
@@ -783,6 +797,60 @@ function refreshCompactionActivity(
|
||||
return changed ? { ...projection, lines } : projection;
|
||||
}
|
||||
|
||||
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
|
||||
if (left.kind !== right.kind) return false;
|
||||
if (left.kind === "idle" || right.kind === "idle") return true;
|
||||
return left.state.kind === right.state.kind &&
|
||||
left.state.state === right.state.state;
|
||||
}
|
||||
|
||||
function workerStateSnapshotEqual(
|
||||
left: WorkerStateSnapshot,
|
||||
right: WorkerStateSnapshot,
|
||||
): boolean {
|
||||
return left.execution_generation === right.execution_generation &&
|
||||
left.revision === right.revision &&
|
||||
left.last_command_id === right.last_command_id &&
|
||||
workerStateEqual(left.state, right.state);
|
||||
}
|
||||
|
||||
function applyWorkerStateSnapshot(
|
||||
projection: ConsoleProjection,
|
||||
incoming: WorkerStateSnapshot,
|
||||
eventId: string,
|
||||
): void {
|
||||
const current = projection.workerState;
|
||||
if (!current) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
const generationOrder = incoming.execution_generation -
|
||||
current.execution_generation;
|
||||
const revisionOrder = incoming.revision - current.revision;
|
||||
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
|
||||
return;
|
||||
}
|
||||
if (!workerStateSnapshotEqual(current, incoming)) {
|
||||
projection.lines.push(
|
||||
line(
|
||||
`${eventId}:worker-state-conflict`,
|
||||
"error",
|
||||
"error · internal",
|
||||
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
|
||||
undefined,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function applyProtocolEvent(
|
||||
projection: ConsoleProjection,
|
||||
envelope: ConsoleEventInput,
|
||||
@@ -793,6 +861,7 @@ export function applyProtocolEvent(
|
||||
tasks: [...projection.tasks],
|
||||
taskNextId: projection.taskNextId,
|
||||
status: projection.status,
|
||||
workerState: projection.workerState,
|
||||
usage: projection.usage,
|
||||
runActivity: applyRunActivityEvent(
|
||||
projection.runActivity,
|
||||
@@ -903,7 +972,6 @@ export function applyProtocolEvent(
|
||||
);
|
||||
break;
|
||||
case "snapshot": {
|
||||
next.status = event.data.status;
|
||||
next.cwd = event.data.greeting.cwd;
|
||||
const snapshot = snapshotProjectionFromSession(
|
||||
envelope.eventId,
|
||||
@@ -953,6 +1021,7 @@ export function applyProtocolEvent(
|
||||
};
|
||||
}
|
||||
}
|
||||
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
|
||||
break;
|
||||
}
|
||||
case "internal_worker": {
|
||||
@@ -1000,8 +1069,15 @@ export function applyProtocolEvent(
|
||||
if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1);
|
||||
break;
|
||||
}
|
||||
case "status":
|
||||
next.status = event.data.status;
|
||||
case "worker_state":
|
||||
applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
|
||||
break;
|
||||
case "command_acknowledged":
|
||||
applyWorkerStateSnapshot(
|
||||
next,
|
||||
event.data.acknowledgement.state,
|
||||
envelope.eventId,
|
||||
);
|
||||
break;
|
||||
case "command":
|
||||
applyCommandEvent(next, envelope.eventId, event.data.event);
|
||||
@@ -1939,6 +2015,7 @@ function snapshotProjectionFromSession(
|
||||
tasks: [],
|
||||
taskNextId: 1,
|
||||
status: null,
|
||||
workerState: null,
|
||||
usage: null,
|
||||
runActivity: emptyRunActivityStats(),
|
||||
cwd,
|
||||
|
||||
@@ -75,7 +75,12 @@ Deno.test("new invoke and running snapshot reset run activity", () => {
|
||||
data: {
|
||||
entries: [],
|
||||
greeting: { text: "", profile: "" },
|
||||
status: "idle",
|
||||
state: {
|
||||
execution_generation: 1,
|
||||
revision: 0,
|
||||
last_command_id: 0,
|
||||
state: { kind: "idle" },
|
||||
},
|
||||
in_flight: {},
|
||||
internal_workers: [],
|
||||
},
|
||||
|
||||
@@ -25,7 +25,9 @@ export function applyRunActivityEvent(
|
||||
case "invoke_start":
|
||||
return { ...emptyRunActivityStats(), startedAtMs: observedAtMs };
|
||||
case "snapshot":
|
||||
return event.data.status === "running"
|
||||
return event.data.state.state.kind === "busy" &&
|
||||
!(event.data.state.state.state.kind === "run" &&
|
||||
event.data.state.state.state.state === "paused")
|
||||
? { ...emptyRunActivityStats(), startedAtMs: observedAtMs }
|
||||
: emptyRunActivityStats();
|
||||
case "turn_start":
|
||||
|
||||
@@ -620,7 +620,7 @@ Deno.test("Worker Console paste chips preserve typed draft and target authority"
|
||||
consolePage.includes("preserveExactText: value.textPastes.length > 0") &&
|
||||
consolePage.includes("composerDrafts.set(activeComposerTargetKey") &&
|
||||
consolePage.includes("switchComposerTarget(target)") &&
|
||||
consolePage.includes('sendControl({ method: "cancel" }, "Stop")'),
|
||||
consolePage.includes('sendWorkerControl("cancel")'),
|
||||
"Paste chips should use shared threshold classification, atomic keyboard behavior, accessible labels, typed restore, and per-Worker draft authority",
|
||||
);
|
||||
});
|
||||
@@ -787,7 +787,10 @@ Deno.test("Worker Console page is routed by runtime_id and worker_id through bac
|
||||
consolePage.includes(
|
||||
'const composerEditable = $derived(protocolState === "open" && !sending);',
|
||||
) &&
|
||||
consolePage.includes('sendControl({ method: "cancel" }, "Stop")') &&
|
||||
consolePage.includes('sendWorkerControl("cancel")') &&
|
||||
consolePage.includes("lifecycleMethod(command)") &&
|
||||
consolePage.includes("expected_worker_state_revision") &&
|
||||
consolePage.includes("expected_execution_generation") &&
|
||||
consolePage.includes("onsubmit={handleComposerSubmit}") &&
|
||||
consolePage.includes("disabled={!composerEditable}") &&
|
||||
consolePage.includes("class:stop={workerRunning}") &&
|
||||
|
||||
@@ -19,6 +19,7 @@ import type {
|
||||
Event as PodProtocolEvent,
|
||||
Method as PodProtocolMethod,
|
||||
Segment as PodProtocolSegment,
|
||||
WorkerStateSnapshot,
|
||||
} from "$lib/generated/protocol";
|
||||
import type {
|
||||
GitCommitSummary as SharedGitCommitSummary,
|
||||
@@ -99,6 +100,7 @@ export type Worker = {
|
||||
tags: string[];
|
||||
workspace: { visibility: string; identity: string };
|
||||
state: string;
|
||||
worker_state?: WorkerStateSnapshot | null;
|
||||
pinned?: boolean;
|
||||
retention_state?: string;
|
||||
last_seen_at?: string | null;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { WorkerStateSnapshot } from "$lib/generated/protocol";
|
||||
|
||||
export function liveWorkerState(worker: {
|
||||
state: string;
|
||||
worker_state?: WorkerStateSnapshot | null;
|
||||
}): string {
|
||||
const state = worker.worker_state?.state;
|
||||
if (!state) return worker.state === "stopped" ? "stopped" : "unknown";
|
||||
if (state.kind === "idle") return "idle";
|
||||
if (state.state.kind === "maintenance") return "running";
|
||||
return state.state.state === "paused" ? "paused" : "running";
|
||||
}
|
||||
@@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void {
|
||||
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
||||
}
|
||||
}
|
||||
import { liveWorkerState } from './worker-state';
|
||||
import {
|
||||
applyWorkspaceWorkersFrame,
|
||||
createWorkspaceWorkersProjection,
|
||||
@@ -33,6 +34,22 @@ function worker(
|
||||
};
|
||||
}
|
||||
|
||||
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
|
||||
const active = worker('runtime-a', 'worker-1', 1);
|
||||
active.worker_state = {
|
||||
execution_generation: 4,
|
||||
revision: 2,
|
||||
last_command_id: 1,
|
||||
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
|
||||
};
|
||||
assertEquals(liveWorkerState(active), 'paused');
|
||||
|
||||
const unavailable = worker('runtime-a', 'worker-2', 1);
|
||||
assertEquals(liveWorkerState(unavailable), 'unknown');
|
||||
unavailable.state = 'stopped';
|
||||
assertEquals(liveWorkerState(unavailable), 'stopped');
|
||||
});
|
||||
|
||||
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
|
||||
const projection = createWorkspaceWorkersProjection();
|
||||
const frame: SubscriptionFrame = {
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
applyWorkspaceWorkersFrame,
|
||||
createWorkspaceWorkersProjection,
|
||||
} from './worker-subscription-model';
|
||||
import { liveWorkerState } from './worker-state';
|
||||
import { compareWorkersForSidebar } from './workers';
|
||||
import type { Worker } from './types';
|
||||
|
||||
@@ -90,7 +91,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker {
|
||||
profile: worker.profile ?? null,
|
||||
tags: [],
|
||||
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
|
||||
state: worker.state,
|
||||
state: liveWorkerState(worker),
|
||||
worker_state: worker.worker_state,
|
||||
pinned: false,
|
||||
retention_state: 'transient',
|
||||
implementation: {
|
||||
|
||||
+60
-31
@@ -52,11 +52,7 @@
|
||||
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
|
||||
import { workspaceApiPath } from "$lib/workspace/api/http";
|
||||
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
|
||||
import type {
|
||||
Diagnostic,
|
||||
Worker,
|
||||
PodProtocolEvent,
|
||||
} from "$lib/workspace/sidebar/types";
|
||||
import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types";
|
||||
|
||||
type Props = {
|
||||
data: {
|
||||
@@ -207,7 +203,6 @@
|
||||
);
|
||||
let pendingObservationEvents: ConsoleEventInput[] = [];
|
||||
let protocolEventSequence = 0;
|
||||
let pendingObservedStates: Array<string | null> = [];
|
||||
let pendingStreamDiagnostics: Diagnostic[] = [];
|
||||
let observationFlushHandle: number | null = null;
|
||||
let nextReloadToken = 0;
|
||||
@@ -249,7 +244,9 @@
|
||||
const diagnostics = $derived(
|
||||
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
|
||||
);
|
||||
const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading");
|
||||
const workerState = $derived(
|
||||
liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"),
|
||||
);
|
||||
const workerRunning = $derived(workerState === "running");
|
||||
const workerPaused = $derived(workerState === "paused");
|
||||
const composerEditable = $derived(protocolState === "open" && !sending);
|
||||
@@ -343,7 +340,6 @@
|
||||
observationFlushHandle = null;
|
||||
}
|
||||
pendingObservationEvents = [];
|
||||
pendingObservedStates = [];
|
||||
pendingStreamDiagnostics = [];
|
||||
}
|
||||
|
||||
@@ -359,18 +355,15 @@
|
||||
function flushObservationBatch() {
|
||||
observationFlushHandle = null;
|
||||
const eventBatch = pendingObservationEvents;
|
||||
const stateBatch = pendingObservedStates;
|
||||
const diagnosticBatch = pendingStreamDiagnostics;
|
||||
pendingObservationEvents = [];
|
||||
pendingObservedStates = [];
|
||||
pendingStreamDiagnostics = [];
|
||||
|
||||
if (eventBatch.length > 0) {
|
||||
const latestState = stateBatch.findLast((state) => state !== null);
|
||||
if (latestState) {
|
||||
liveWorkerState = latestState;
|
||||
}
|
||||
consoleProjection = consoleProjector.append(eventBatch);
|
||||
liveWorkerState = consoleProjection.status === "shutdown"
|
||||
? "shutdown"
|
||||
: workerStateFromSnapshot(consoleProjection.workerState);
|
||||
advanceEventObservedAtVersion();
|
||||
}
|
||||
|
||||
@@ -407,7 +400,6 @@
|
||||
event: payload,
|
||||
observedAtMs,
|
||||
});
|
||||
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
|
||||
scheduleObservationFlush();
|
||||
}
|
||||
|
||||
@@ -541,9 +533,42 @@
|
||||
}
|
||||
}
|
||||
|
||||
let nextWorkerCommandId = 1;
|
||||
|
||||
function lifecycleMethod(
|
||||
command: "pause" | "cancel" | "resume" | "compact",
|
||||
): ProtocolMethod | null {
|
||||
const state = consoleProjection.workerState;
|
||||
if (!state) {
|
||||
sendError = "Worker state snapshot is not available; reconnect before sending control.";
|
||||
return null;
|
||||
}
|
||||
const commandId = Math.max(
|
||||
nextWorkerCommandId,
|
||||
state.last_command_id + 1,
|
||||
);
|
||||
nextWorkerCommandId = commandId + 1;
|
||||
const envelope = {
|
||||
command_id: commandId,
|
||||
expected_execution_generation: state.execution_generation,
|
||||
expected_worker_state_revision: state.revision,
|
||||
};
|
||||
switch (command) {
|
||||
case "pause":
|
||||
return { method: "pause", params: { command: envelope } };
|
||||
case "cancel":
|
||||
return { method: "cancel", params: { command: envelope } };
|
||||
case "resume":
|
||||
return { method: "resume", params: { command: envelope } };
|
||||
case "compact":
|
||||
return { method: "compact", params: { command: envelope } };
|
||||
}
|
||||
}
|
||||
|
||||
function sendWorkerControl(command: "pause" | "cancel" | "resume") {
|
||||
const label = command[0].toUpperCase() + command.slice(1);
|
||||
sendControl({ method: command }, label);
|
||||
const method = lifecycleMethod(command);
|
||||
if (method) sendControl(method, label);
|
||||
}
|
||||
|
||||
function isEditableTarget(target: EventTarget | null): boolean {
|
||||
@@ -627,8 +652,11 @@
|
||||
auto_run: true,
|
||||
},
|
||||
};
|
||||
case "compact":
|
||||
return { method: "compact" };
|
||||
case "compact": {
|
||||
const method = lifecycleMethod("compact");
|
||||
if (!method) throw new Error("Worker state snapshot is not available");
|
||||
return method;
|
||||
}
|
||||
case "list_rewind_targets":
|
||||
return { method: "list_rewind_targets" };
|
||||
case "register_peer":
|
||||
@@ -691,7 +719,7 @@
|
||||
|
||||
function handleComposerSubmit() {
|
||||
if (workerRunning) {
|
||||
sendControl({ method: "cancel" }, "Stop");
|
||||
sendWorkerControl("cancel");
|
||||
return;
|
||||
}
|
||||
void submitDraft(composerInputElement?.snapshot() ?? draft);
|
||||
@@ -889,18 +917,16 @@
|
||||
handleComposerSubmit();
|
||||
}
|
||||
|
||||
function workerStateFromProtocolEvent(
|
||||
event: PodProtocolEvent,
|
||||
function workerStateFromSnapshot(
|
||||
snapshot: ConsoleProjection["workerState"],
|
||||
): string | null {
|
||||
switch (event.event) {
|
||||
case "snapshot":
|
||||
case "status":
|
||||
return event.data.status;
|
||||
case "shutdown":
|
||||
return "shutdown";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
if (!snapshot) return null;
|
||||
return snapshot.state.kind === "idle"
|
||||
? "idle"
|
||||
: snapshot.state.state.kind === "run" &&
|
||||
snapshot.state.state.state === "paused"
|
||||
? "paused"
|
||||
: "running";
|
||||
}
|
||||
|
||||
function connectProtocolTransport(
|
||||
@@ -1620,7 +1646,10 @@
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={protocolState !== "open"}
|
||||
onclick={() => sendControl({ method: "compact" }, "Compact")}
|
||||
onclick={() => {
|
||||
const method = lifecycleMethod("compact");
|
||||
if (method) sendControl(method, "Compact");
|
||||
}}
|
||||
>
|
||||
Compact
|
||||
</button>
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { workerHref } from '$lib/workspace/resource-links';
|
||||
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
|
||||
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
|
||||
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
|
||||
import type { PageProps } from './$types';
|
||||
|
||||
@@ -136,7 +137,7 @@
|
||||
}
|
||||
|
||||
function workerStatus(worker: Worker): string {
|
||||
return worker.state;
|
||||
return liveWorkerState(worker);
|
||||
}
|
||||
|
||||
function workerProfile(worker: Worker): string {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { workspaceRoute } from '$lib/workspace/api/http';
|
||||
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||
import type { PageData } from './$types';
|
||||
let { data }: { data: PageData } = $props();
|
||||
</script>
|
||||
@@ -24,7 +25,7 @@
|
||||
>Open console</a>
|
||||
</header>
|
||||
<dl class="resource-meta">
|
||||
<dt>Status</dt><dd>{data.worker.state}</dd>
|
||||
<dt>Status</dt><dd>{liveWorkerState(data.worker)}</dd>
|
||||
<dt>Profile</dt><dd>{data.worker.profile}</dd>
|
||||
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
|
||||
</dl>
|
||||
|
||||
Reference in New Issue
Block a user