feat: integrate Worker state authority
This commit is contained in:
@@ -112,8 +112,8 @@ mod tests {
|
|||||||
async fn encodes_methods_and_decodes_events_above_transport() {
|
async fn encodes_methods_and_decodes_events_above_transport() {
|
||||||
let mut socket = TestSocket::default();
|
let mut socket = TestSocket::default();
|
||||||
socket.incoming.push_back(
|
socket.incoming.push_back(
|
||||||
encode_event(&Event::Status {
|
encode_event(&Event::WorkerState {
|
||||||
status: WorkerStatus::Idle,
|
snapshot: WorkerStatus::Idle.into(),
|
||||||
})
|
})
|
||||||
.expect("encode event"),
|
.expect("encode event"),
|
||||||
);
|
);
|
||||||
@@ -132,9 +132,7 @@ mod tests {
|
|||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
client.next_event().await,
|
client.next_event().await,
|
||||||
Ok(Some(Event::Status {
|
Ok(Some(Event::WorkerState { .. }))
|
||||||
status: WorkerStatus::Idle
|
|
||||||
}))
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -101,8 +101,8 @@ mod tests {
|
|||||||
));
|
));
|
||||||
|
|
||||||
peer.send(
|
peer.send(
|
||||||
encode_event(&Event::Status {
|
encode_event(&Event::WorkerState {
|
||||||
status: WorkerStatus::Idle,
|
snapshot: WorkerStatus::Idle.into(),
|
||||||
})
|
})
|
||||||
.expect("encode event"),
|
.expect("encode event"),
|
||||||
)
|
)
|
||||||
@@ -110,9 +110,7 @@ mod tests {
|
|||||||
.expect("send event");
|
.expect("send event");
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
client.next_event().await,
|
client.next_event().await,
|
||||||
Ok(Some(Event::Status {
|
Ok(Some(Event::WorkerState { .. }))
|
||||||
status: WorkerStatus::Idle
|
|
||||||
}))
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -113,8 +113,8 @@ mod tests {
|
|||||||
let listener = UnixListener::bind(&socket_path).unwrap();
|
let listener = UnixListener::bind(&socket_path).unwrap();
|
||||||
let server = tokio::spawn(async move {
|
let server = tokio::spawn(async move {
|
||||||
let (mut stream, _) = listener.accept().await.unwrap();
|
let (mut stream, _) = listener.accept().await.unwrap();
|
||||||
let event = encode_event(&Event::Status {
|
let event = encode_event(&Event::WorkerState {
|
||||||
status: WorkerStatus::Idle,
|
snapshot: WorkerStatus::Idle.into(),
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
stream.write_all(event.as_bytes()).await.unwrap();
|
stream.write_all(event.as_bytes()).await.unwrap();
|
||||||
@@ -126,12 +126,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect("client should receive event while alive")
|
.expect("client should receive event while alive")
|
||||||
.expect("transport should succeed");
|
.expect("transport should succeed");
|
||||||
assert!(matches!(
|
assert!(matches!(event, Some(Event::WorkerState { .. })));
|
||||||
event,
|
|
||||||
Some(Event::Status {
|
|
||||||
status: WorkerStatus::Idle
|
|
||||||
})
|
|
||||||
));
|
|
||||||
server.await.unwrap();
|
server.await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -116,8 +116,8 @@ mod tests {
|
|||||||
Message::Text(ref text)
|
Message::Text(ref text)
|
||||||
if matches!(decode_method(text), Ok(Method::Submit { .. }))
|
if matches!(decode_method(text), Ok(Method::Submit { .. }))
|
||||||
));
|
));
|
||||||
let event = encode_event(&Event::Status {
|
let event = encode_event(&Event::WorkerState {
|
||||||
status: WorkerStatus::Idle,
|
snapshot: WorkerStatus::Idle.into(),
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
socket.send(Message::Text(event.into())).await.unwrap();
|
socket.send(Message::Text(event.into())).await.unwrap();
|
||||||
@@ -134,9 +134,7 @@ mod tests {
|
|||||||
.expect("send method");
|
.expect("send method");
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
client.next_event().await,
|
client.next_event().await,
|
||||||
Ok(Some(Event::Status {
|
Ok(Some(Event::WorkerState { .. }))
|
||||||
status: WorkerStatus::Idle
|
|
||||||
}))
|
|
||||||
));
|
));
|
||||||
server.await.unwrap();
|
server.await.unwrap();
|
||||||
}
|
}
|
||||||
|
|||||||
+321
-51
@@ -85,6 +85,190 @@ impl AuthenticatedInputSource {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Immutable identity and revision fence for one state-changing Worker command.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
pub struct WorkerCommandEnvelope {
|
||||||
|
/// Caller-owned sequence. A controller accepts command ids in strictly
|
||||||
|
/// increasing order for one execution generation.
|
||||||
|
pub command_id: u64,
|
||||||
|
pub expected_execution_generation: u64,
|
||||||
|
pub expected_worker_state_revision: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerCommandEnvelope {
|
||||||
|
pub fn for_snapshot(command_id: u64, snapshot: &WorkerStateSnapshot) -> Self {
|
||||||
|
Self {
|
||||||
|
command_id,
|
||||||
|
expected_execution_generation: snapshot.execution_generation,
|
||||||
|
expected_worker_state_revision: snapshot.revision,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum WorkerCommandKind {
|
||||||
|
Resume,
|
||||||
|
Cancel,
|
||||||
|
Pause,
|
||||||
|
Compact,
|
||||||
|
Shutdown,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum WorkerCommandDisposition {
|
||||||
|
Accepted,
|
||||||
|
StaleExecutionGeneration,
|
||||||
|
StaleWorkerStateRevision,
|
||||||
|
StaleCommandId,
|
||||||
|
Conflict,
|
||||||
|
InvalidState,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
pub struct WorkerCommandAcknowledgement {
|
||||||
|
pub command_id: u64,
|
||||||
|
pub command: WorkerCommandKind,
|
||||||
|
pub disposition: WorkerCommandDisposition,
|
||||||
|
/// The complete authoritative state observed after command admission.
|
||||||
|
pub state: WorkerStateSnapshot,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(tag = "kind", content = "state", rename_all = "snake_case")]
|
||||||
|
pub enum WorkerState {
|
||||||
|
Idle,
|
||||||
|
Busy(WorkerBusyState),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(tag = "kind", content = "state", rename_all = "snake_case")]
|
||||||
|
pub enum WorkerBusyState {
|
||||||
|
Run(WorkerRunState),
|
||||||
|
Maintenance(WorkerMaintenanceState),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum WorkerRunState {
|
||||||
|
Running,
|
||||||
|
Pausing,
|
||||||
|
Paused,
|
||||||
|
Cancelling,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum WorkerMaintenanceState {
|
||||||
|
Compacting,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
pub struct WorkerStateSnapshot {
|
||||||
|
pub execution_generation: u64,
|
||||||
|
pub revision: u64,
|
||||||
|
/// Highest lifecycle command id observed by this controller generation.
|
||||||
|
pub last_command_id: u64,
|
||||||
|
pub state: WorkerState,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerStateSnapshot {
|
||||||
|
pub fn initial(execution_generation: u64) -> Self {
|
||||||
|
Self {
|
||||||
|
execution_generation,
|
||||||
|
revision: 0,
|
||||||
|
last_command_id: 0,
|
||||||
|
state: WorkerState::Idle,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compatibility projection for Runtime catalog lifecycle. This value is
|
||||||
|
/// never command-admission authority and cannot produce `Stopped`.
|
||||||
|
pub fn catalog_status(&self) -> WorkerStatus {
|
||||||
|
match self.state {
|
||||||
|
WorkerState::Idle => WorkerStatus::Idle,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(_))
|
||||||
|
| WorkerState::Busy(WorkerBusyState::Maintenance(_)) => WorkerStatus::Running,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum WorkerStateSnapshotApply {
|
||||||
|
Applied,
|
||||||
|
Duplicate,
|
||||||
|
Stale,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct WorkerStateSnapshotConflict {
|
||||||
|
pub execution_generation: u64,
|
||||||
|
pub revision: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for WorkerStateSnapshotConflict {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
formatter,
|
||||||
|
"conflicting worker state snapshots at generation {} revision {}",
|
||||||
|
self.execution_generation, self.revision
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for WorkerStateSnapshotConflict {}
|
||||||
|
|
||||||
|
pub fn apply_worker_state_snapshot(
|
||||||
|
current: &mut WorkerStateSnapshot,
|
||||||
|
incoming: &WorkerStateSnapshot,
|
||||||
|
) -> Result<WorkerStateSnapshotApply, WorkerStateSnapshotConflict> {
|
||||||
|
use std::cmp::Ordering;
|
||||||
|
|
||||||
|
let ordering = (incoming.execution_generation, incoming.revision)
|
||||||
|
.cmp(&(current.execution_generation, current.revision));
|
||||||
|
match ordering {
|
||||||
|
Ordering::Greater => {
|
||||||
|
*current = incoming.clone();
|
||||||
|
Ok(WorkerStateSnapshotApply::Applied)
|
||||||
|
}
|
||||||
|
Ordering::Less => Ok(WorkerStateSnapshotApply::Stale),
|
||||||
|
Ordering::Equal if incoming == current => Ok(WorkerStateSnapshotApply::Duplicate),
|
||||||
|
Ordering::Equal => Err(WorkerStateSnapshotConflict {
|
||||||
|
execution_generation: incoming.execution_generation,
|
||||||
|
revision: incoming.revision,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<WorkerStatus> for WorkerStateSnapshot {
|
||||||
|
fn from(status: WorkerStatus) -> Self {
|
||||||
|
let state = match status {
|
||||||
|
WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
|
||||||
|
WorkerStatus::Running => {
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||||
|
}
|
||||||
|
WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
|
||||||
|
};
|
||||||
|
Self {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 0,
|
||||||
|
last_command_id: 0,
|
||||||
|
state,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(tag = "method", content = "params", rename_all = "snake_case")]
|
#[serde(tag = "method", content = "params", rename_all = "snake_case")]
|
||||||
@@ -149,20 +333,28 @@ pub enum Method {
|
|||||||
expected_revision: u64,
|
expected_revision: u64,
|
||||||
expected_head_id: String,
|
expected_head_id: String,
|
||||||
},
|
},
|
||||||
Resume,
|
Resume {
|
||||||
Cancel,
|
command: WorkerCommandEnvelope,
|
||||||
|
},
|
||||||
|
Cancel {
|
||||||
|
command: WorkerCommandEnvelope,
|
||||||
|
},
|
||||||
/// Stop the in-flight turn and transition to `Paused`.
|
/// Stop the in-flight turn and transition to `Paused`.
|
||||||
///
|
///
|
||||||
/// Unlike `Cancel` (which discards and returns to `Idle`), a paused
|
/// Unlike `Cancel` (which discards and returns to `Idle`), a paused
|
||||||
/// Worker can resume the interrupted work via `Resume`, or accept a
|
/// Worker can resume the interrupted work via `Resume`, or accept a
|
||||||
/// fresh `Submit` (orphan `tool_use` items are closed with a
|
/// fresh `Submit` (orphan `tool_use` items are closed with a
|
||||||
/// synthetic tool result before the new user message is appended).
|
/// synthetic tool result before the new user message is appended).
|
||||||
Pause,
|
Pause {
|
||||||
|
command: WorkerCommandEnvelope,
|
||||||
|
},
|
||||||
/// Request an explicit compaction while the Worker is otherwise idle.
|
/// Request an explicit compaction while the Worker is otherwise idle.
|
||||||
///
|
///
|
||||||
/// This is a typed control method: clients must not send `compact` as a
|
/// This is a typed control method: clients must not send `compact` as a
|
||||||
/// `Method::Submit` user message.
|
/// `Method::Submit` user message.
|
||||||
Compact,
|
Compact {
|
||||||
|
command: WorkerCommandEnvelope,
|
||||||
|
},
|
||||||
/// Ask the Worker to list valid rewind targets from its authoritative session log.
|
/// Ask the Worker to list valid rewind targets from its authoritative session log.
|
||||||
ListRewindTargets,
|
ListRewindTargets,
|
||||||
/// Truncate the current session back to the selected rewind target and
|
/// Truncate the current session back to the selected rewind target and
|
||||||
@@ -171,7 +363,9 @@ pub enum Method {
|
|||||||
target: RewindTargetId,
|
target: RewindTargetId,
|
||||||
expected_head_entries: usize,
|
expected_head_entries: usize,
|
||||||
},
|
},
|
||||||
Shutdown,
|
Shutdown {
|
||||||
|
command: WorkerCommandEnvelope,
|
||||||
|
},
|
||||||
/// Request a list of completion candidates from the Worker.
|
/// Request a list of completion candidates from the Worker.
|
||||||
///
|
///
|
||||||
/// Reply is sent on the same socket as `Event::Completions` (not
|
/// Reply is sent on the same socket as `Event::Completions` (not
|
||||||
@@ -938,8 +1132,9 @@ pub enum Event {
|
|||||||
Snapshot {
|
Snapshot {
|
||||||
session: SessionSnapshot,
|
session: SessionSnapshot,
|
||||||
greeting: Greeting,
|
greeting: Greeting,
|
||||||
#[serde(default)]
|
/// Full revisioned live execution state. `Stopped` remains Runtime
|
||||||
status: WorkerStatus,
|
/// catalog authority and is deliberately not represented here.
|
||||||
|
state: WorkerStateSnapshot,
|
||||||
/// Unfinished model output that has already streamed in the current
|
/// Unfinished model output that has already streamed in the current
|
||||||
/// run but is not yet represented by committed snapshot entries.
|
/// run but is not yet represented by committed snapshot entries.
|
||||||
#[serde(default, skip_serializing_if = "InFlightSnapshot::is_empty")]
|
#[serde(default, skip_serializing_if = "InFlightSnapshot::is_empty")]
|
||||||
@@ -976,8 +1171,11 @@ pub enum Event {
|
|||||||
},
|
},
|
||||||
/// Current Worker controller status. Broadcast on every controller-level
|
/// Current Worker controller status. Broadcast on every controller-level
|
||||||
/// transition and included in `History` snapshots for late attach.
|
/// transition and included in `History` snapshots for late attach.
|
||||||
Status {
|
WorkerState {
|
||||||
status: WorkerStatus,
|
snapshot: WorkerStateSnapshot,
|
||||||
|
},
|
||||||
|
CommandAcknowledged {
|
||||||
|
acknowledgement: WorkerCommandAcknowledgement,
|
||||||
},
|
},
|
||||||
/// Bounded, provider-owned command telemetry for the live Console. This is
|
/// Bounded, provider-owned command telemetry for the live Console. This is
|
||||||
/// intentionally not a history entry and is reconstructed from
|
/// intentionally not a history entry and is reconstructed from
|
||||||
@@ -1424,6 +1622,58 @@ pub enum Permission {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_state_snapshot_apply_is_monotonic_and_detects_conflicts() {
|
||||||
|
let mut current = WorkerStateSnapshot::initial(4);
|
||||||
|
let mut newer = current.clone();
|
||||||
|
newer.revision = 1;
|
||||||
|
newer.state = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
apply_worker_state_snapshot(&mut current, &newer),
|
||||||
|
Ok(WorkerStateSnapshotApply::Applied)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
apply_worker_state_snapshot(&mut current, &newer),
|
||||||
|
Ok(WorkerStateSnapshotApply::Duplicate)
|
||||||
|
);
|
||||||
|
|
||||||
|
let stale_revision = WorkerStateSnapshot::initial(4);
|
||||||
|
assert_eq!(
|
||||||
|
apply_worker_state_snapshot(&mut current, &stale_revision),
|
||||||
|
Ok(WorkerStateSnapshotApply::Stale)
|
||||||
|
);
|
||||||
|
let stale_generation = WorkerStateSnapshot {
|
||||||
|
execution_generation: 3,
|
||||||
|
revision: u64::MAX,
|
||||||
|
..newer.clone()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
apply_worker_state_snapshot(&mut current, &stale_generation),
|
||||||
|
Ok(WorkerStateSnapshotApply::Stale)
|
||||||
|
);
|
||||||
|
|
||||||
|
let conflicting = WorkerStateSnapshot {
|
||||||
|
state: WorkerState::Idle,
|
||||||
|
..newer.clone()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
apply_worker_state_snapshot(&mut current, &conflicting),
|
||||||
|
Err(WorkerStateSnapshotConflict {
|
||||||
|
execution_generation: 4,
|
||||||
|
revision: 1,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(current, newer);
|
||||||
|
|
||||||
|
let next_generation = WorkerStateSnapshot::initial(5);
|
||||||
|
assert_eq!(
|
||||||
|
apply_worker_state_snapshot(&mut current, &next_generation),
|
||||||
|
Ok(WorkerStateSnapshotApply::Applied)
|
||||||
|
);
|
||||||
|
assert_eq!(current, next_generation);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn method_submit_json_roundtrip_and_run_is_rejected() {
|
fn method_submit_json_roundtrip_and_run_is_rejected() {
|
||||||
let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#;
|
let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#;
|
||||||
@@ -1612,28 +1862,39 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn method_without_params() {
|
fn lifecycle_method_without_command_fails_closed() {
|
||||||
let json = r#"{"method":"resume"}"#;
|
let error = serde_json::from_str::<Method>(r#"{"method":"resume"}"#).unwrap_err();
|
||||||
let method: Method = serde_json::from_str(json).unwrap();
|
assert!(error.to_string().contains("params"));
|
||||||
assert!(matches!(method, Method::Resume));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn method_pause_roundtrip() {
|
fn lifecycle_methods_roundtrip_with_fences() {
|
||||||
let json = r#"{"method":"pause"}"#;
|
for method in [
|
||||||
let method: Method = serde_json::from_str(json).unwrap();
|
Method::Pause {
|
||||||
assert!(matches!(method, Method::Pause));
|
command: WorkerCommandEnvelope {
|
||||||
let serialized = serde_json::to_string(&method).unwrap();
|
command_id: 11,
|
||||||
assert_eq!(serialized, json);
|
expected_execution_generation: 4,
|
||||||
|
expected_worker_state_revision: 8,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Method::Compact {
|
||||||
|
command: WorkerCommandEnvelope {
|
||||||
|
command_id: 12,
|
||||||
|
expected_execution_generation: 4,
|
||||||
|
expected_worker_state_revision: 9,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
let json = serde_json::to_string(&method).unwrap();
|
||||||
|
let decoded: Method = serde_json::from_str(&json).unwrap();
|
||||||
|
match decoded {
|
||||||
|
Method::Pause { command } | Method::Compact { command } => {
|
||||||
|
assert_eq!(command.expected_execution_generation, 4);
|
||||||
|
assert!(command.command_id >= 11);
|
||||||
|
}
|
||||||
|
other => panic!("unexpected lifecycle method: {other:?}"),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn method_compact_roundtrip() {
|
|
||||||
let json = r#"{"method":"compact"}"#;
|
|
||||||
let method: Method = serde_json::from_str(json).unwrap();
|
|
||||||
assert!(matches!(method, Method::Compact));
|
|
||||||
let serialized = serde_json::to_string(&method).unwrap();
|
|
||||||
assert_eq!(serialized, json);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1902,7 +2163,7 @@ mod tests {
|
|||||||
context_window: 200_000,
|
context_window: 200_000,
|
||||||
context_tokens: 42_000,
|
context_tokens: 42_000,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Paused,
|
state: WorkerStatus::Paused.into(),
|
||||||
in_flight: InFlightSnapshot::default(),
|
in_flight: InFlightSnapshot::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
};
|
};
|
||||||
@@ -1919,12 +2180,13 @@ mod tests {
|
|||||||
assert_eq!(parsed["data"]["greeting"]["tools"][0], "Read");
|
assert_eq!(parsed["data"]["greeting"]["tools"][0], "Read");
|
||||||
assert_eq!(parsed["data"]["greeting"]["context_window"], 200_000);
|
assert_eq!(parsed["data"]["greeting"]["context_window"], 200_000);
|
||||||
assert_eq!(parsed["data"]["greeting"]["context_tokens"], 42_000);
|
assert_eq!(parsed["data"]["greeting"]["context_tokens"], 42_000);
|
||||||
assert_eq!(parsed["data"]["status"], "paused");
|
assert_eq!(parsed["data"]["state"]["state"]["kind"], "busy");
|
||||||
|
assert_eq!(parsed["data"]["state"]["state"]["state"]["state"], "paused");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn event_snapshot_in_flight_roundtrip_and_default() {
|
fn event_snapshot_in_flight_roundtrip_and_default() {
|
||||||
let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"status":"running"}}"#;
|
let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"state":{"execution_generation":1,"revision":1,"last_command_id":0,"state":{"kind":"busy","state":{"kind":"run","state":"running"}}}}}"#;
|
||||||
let decoded: Event = serde_json::from_str(inbound).unwrap();
|
let decoded: Event = serde_json::from_str(inbound).unwrap();
|
||||||
match decoded {
|
match decoded {
|
||||||
Event::Snapshot { in_flight, .. } => assert!(in_flight.is_empty()),
|
Event::Snapshot { in_flight, .. } => assert!(in_flight.is_empty()),
|
||||||
@@ -1946,7 +2208,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Running,
|
state: WorkerStatus::Running.into(),
|
||||||
in_flight: InFlightSnapshot {
|
in_flight: InFlightSnapshot {
|
||||||
blocks: vec![
|
blocks: vec![
|
||||||
InFlightBlock::Text {
|
InFlightBlock::Text {
|
||||||
@@ -2034,20 +2296,32 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn event_status_format() {
|
fn event_worker_state_format() {
|
||||||
let event = Event::Status {
|
let event = Event::WorkerState {
|
||||||
status: WorkerStatus::Running,
|
snapshot: WorkerStateSnapshot {
|
||||||
|
execution_generation: 7,
|
||||||
|
revision: 3,
|
||||||
|
last_command_id: 9,
|
||||||
|
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
let json = serde_json::to_string(&event).unwrap();
|
let json = serde_json::to_string(&event).unwrap();
|
||||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||||
assert_eq!(parsed["event"], "status");
|
assert_eq!(parsed["event"], "worker_state");
|
||||||
assert_eq!(parsed["data"]["status"], "running");
|
assert_eq!(parsed["data"]["snapshot"]["execution_generation"], 7);
|
||||||
|
assert_eq!(parsed["data"]["snapshot"]["revision"], 3);
|
||||||
|
assert_eq!(parsed["data"]["snapshot"]["state"]["kind"], "busy");
|
||||||
|
|
||||||
let decoded: Event = serde_json::from_str(&json).unwrap();
|
let decoded: Event = serde_json::from_str(&json).unwrap();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
decoded,
|
decoded,
|
||||||
Event::Status {
|
Event::WorkerState {
|
||||||
status: WorkerStatus::Running
|
snapshot: WorkerStateSnapshot {
|
||||||
|
execution_generation: 7,
|
||||||
|
revision: 3,
|
||||||
|
state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||||
|
..
|
||||||
|
}
|
||||||
}
|
}
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@@ -2088,19 +2362,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn event_snapshot_without_status_defaults_to_idle() {
|
fn event_snapshot_without_worker_state_fails_closed() {
|
||||||
let json = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#;
|
let json = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#;
|
||||||
let decoded: Event = serde_json::from_str(json).unwrap();
|
let error = serde_json::from_str::<Event>(json).unwrap_err();
|
||||||
match decoded {
|
assert!(error.to_string().contains("state"));
|
||||||
Event::Snapshot {
|
|
||||||
status, greeting, ..
|
|
||||||
} => {
|
|
||||||
assert_eq!(status, WorkerStatus::Idle);
|
|
||||||
assert_eq!(greeting.context_window, 0);
|
|
||||||
assert_eq!(greeting.context_tokens, 0);
|
|
||||||
}
|
|
||||||
other => panic!("expected Snapshot, got {other:?}"),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2513,7 +2778,12 @@ mod tests {
|
|||||||
"scope_summary": "scope",
|
"scope_summary": "scope",
|
||||||
"tools": []
|
"tools": []
|
||||||
},
|
},
|
||||||
"status": "idle"
|
"state": {
|
||||||
|
"execution_generation": 1,
|
||||||
|
"revision": 0,
|
||||||
|
"last_command_id": 0,
|
||||||
|
"state": { "kind": "idle" }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|||||||
@@ -573,6 +573,11 @@ pub struct SubscriptionWorker {
|
|||||||
pub resource_key: Option<String>,
|
pub resource_key: Option<String>,
|
||||||
/// Producer-owned monotonic revision for this Worker subject.
|
/// Producer-owned monotonic revision for this Worker subject.
|
||||||
pub subject_revision: u64,
|
pub subject_revision: u64,
|
||||||
|
/// Latest revisioned foreground state observed from the Worker. This remains
|
||||||
|
/// absent until an authoritative Worker snapshot/event has been applied.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<crate::WorkerStateSnapshot>,
|
||||||
|
/// Runtime catalog lifecycle compatibility projection; not foreground-state authority.
|
||||||
pub state: SubscriptionWorkerState,
|
pub state: SubscriptionWorkerState,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub has_running_internal_workers: bool,
|
pub has_running_internal_workers: bool,
|
||||||
@@ -874,6 +879,7 @@ mod tests {
|
|||||||
runtime_id: None,
|
runtime_id: None,
|
||||||
resource_key: None,
|
resource_key: None,
|
||||||
subject_revision: 0,
|
subject_revision: 0,
|
||||||
|
worker_state: None,
|
||||||
state: SubscriptionWorkerState::Idle,
|
state: SubscriptionWorkerState::Idle,
|
||||||
has_running_internal_workers: false,
|
has_running_internal_workers: false,
|
||||||
workspace_id: Some("workspace-1".to_string()),
|
workspace_id: Some("workspace-1".to_string()),
|
||||||
|
|||||||
@@ -12,7 +12,10 @@ use crate::{
|
|||||||
RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart,
|
RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart,
|
||||||
SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry,
|
SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry,
|
||||||
SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition,
|
SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition,
|
||||||
TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, WorkerStatus,
|
TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerBusyState,
|
||||||
|
WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope,
|
||||||
|
WorkerCommandKind, WorkerEvent, WorkerMaintenanceState, WorkerRunState, WorkerState,
|
||||||
|
WorkerStateSnapshot, WorkerStatus,
|
||||||
subscription::{
|
subscription::{
|
||||||
EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame,
|
EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame,
|
||||||
SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest,
|
SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest,
|
||||||
@@ -46,6 +49,15 @@ pub fn generated_protocol_types() -> String {
|
|||||||
push_decl::<AlertSource>(&cfg, &mut output);
|
push_decl::<AlertSource>(&cfg, &mut output);
|
||||||
push_decl::<CompletionKind>(&cfg, &mut output);
|
push_decl::<CompletionKind>(&cfg, &mut output);
|
||||||
push_decl::<WorkerStatus>(&cfg, &mut output);
|
push_decl::<WorkerStatus>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerCommandEnvelope>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerCommandKind>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerCommandDisposition>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerCommandAcknowledgement>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerRunState>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerMaintenanceState>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerBusyState>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerState>(&cfg, &mut output);
|
||||||
|
push_decl::<WorkerStateSnapshot>(&cfg, &mut output);
|
||||||
push_decl::<TurnResult>(&cfg, &mut output);
|
push_decl::<TurnResult>(&cfg, &mut output);
|
||||||
push_decl::<InvokeKind>(&cfg, &mut output);
|
push_decl::<InvokeKind>(&cfg, &mut output);
|
||||||
push_decl::<RunResult>(&cfg, &mut output);
|
push_decl::<RunResult>(&cfg, &mut output);
|
||||||
|
|||||||
@@ -318,7 +318,11 @@ impl StandaloneHost {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> {
|
pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> {
|
||||||
let _ = self.handle.send(Method::Shutdown).await;
|
let command = protocol::WorkerCommandEnvelope::for_snapshot(
|
||||||
|
u64::MAX,
|
||||||
|
&self.handle.shared_state.snapshot(),
|
||||||
|
);
|
||||||
|
let _ = self.handle.send(Method::Shutdown { command }).await;
|
||||||
let Some(shutdown) = self.shutdown.take() else {
|
let Some(shutdown) = self.shutdown.take() else {
|
||||||
self.retain_lease();
|
self.retain_lease();
|
||||||
return Err(StandaloneShutdownError::ConfirmationLost);
|
return Err(StandaloneShutdownError::ConfirmationLost);
|
||||||
@@ -500,7 +504,11 @@ fn active_pointer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn stop_started_worker(started: BootstrappedWorker) {
|
async fn stop_started_worker(started: BootstrappedWorker) {
|
||||||
let _ = started.handle.send(Method::Shutdown).await;
|
let command = protocol::WorkerCommandEnvelope::for_snapshot(
|
||||||
|
u64::MAX,
|
||||||
|
&started.handle.shared_state.snapshot(),
|
||||||
|
);
|
||||||
|
let _ = started.handle.send(Method::Shutdown { command }).await;
|
||||||
let _ = tokio::time::timeout(Duration::from_secs(2), started.shutdown).await;
|
let _ = tokio::time::timeout(Duration::from_secs(2), started.shutdown).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+158
-38
@@ -5,7 +5,7 @@ use std::time::{Duration, Instant};
|
|||||||
use protocol::{
|
use protocol::{
|
||||||
AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, InFlightBlock,
|
AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, InFlightBlock,
|
||||||
InFlightSnapshot, InFlightToolCallState, InternalWorkerRef, InternalWorkerSnapshot, Method,
|
InFlightSnapshot, InFlightToolCallState, InternalWorkerRef, InternalWorkerSnapshot, Method,
|
||||||
RewindTarget, RunResult, Segment, WorkerStatus,
|
RewindTarget, RunResult, Segment, WorkerCommandEnvelope, WorkerStateSnapshot, WorkerStatus,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::block::{
|
use crate::block::{
|
||||||
@@ -225,8 +225,10 @@ pub struct WorkerViewTab {
|
|||||||
pub struct App {
|
pub struct App {
|
||||||
pub worker_name: String,
|
pub worker_name: String,
|
||||||
pub connected: bool,
|
pub connected: bool,
|
||||||
/// Last controller status reported by the Worker. Drives the status line
|
/// Latest authoritative revisioned live execution state.
|
||||||
/// and Ctrl-key routing; do not infer this solely from replayed history.
|
pub worker_state: WorkerStateSnapshot,
|
||||||
|
next_command_id: u64,
|
||||||
|
/// Derived Runtime-catalog compatibility projection used by existing UI.
|
||||||
pub worker_status: WorkerStatus,
|
pub worker_status: WorkerStatus,
|
||||||
/// True while the Worker is in `WorkerStatus::Running`.
|
/// True while the Worker is in `WorkerStatus::Running`.
|
||||||
pub running: bool,
|
pub running: bool,
|
||||||
@@ -337,6 +339,8 @@ impl App {
|
|||||||
Self {
|
Self {
|
||||||
worker_name,
|
worker_name,
|
||||||
connected: false,
|
connected: false,
|
||||||
|
worker_state: WorkerStateSnapshot::initial(1),
|
||||||
|
next_command_id: 1,
|
||||||
worker_status: WorkerStatus::Idle,
|
worker_status: WorkerStatus::Idle,
|
||||||
running: false,
|
running: false,
|
||||||
paused: false,
|
paused: false,
|
||||||
@@ -745,7 +749,8 @@ impl App {
|
|||||||
if self.paused {
|
if self.paused {
|
||||||
self.input_history.cancel_browse();
|
self.input_history.cancel_browse();
|
||||||
self.input.clear();
|
self.input.clear();
|
||||||
return Some(Method::Resume);
|
let command = self.next_command_envelope();
|
||||||
|
return Some(Method::Resume { command });
|
||||||
}
|
}
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -1114,6 +1119,31 @@ impl App {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn next_command_envelope(&mut self) -> WorkerCommandEnvelope {
|
||||||
|
let command_id = self
|
||||||
|
.next_command_id
|
||||||
|
.max(self.worker_state.last_command_id.saturating_add(1));
|
||||||
|
let command = WorkerCommandEnvelope::for_snapshot(command_id, &self.worker_state);
|
||||||
|
self.next_command_id = command_id.saturating_add(1);
|
||||||
|
command
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_worker_state_snapshot(&mut self, snapshot: &WorkerStateSnapshot) {
|
||||||
|
match protocol::apply_worker_state_snapshot(&mut self.worker_state, snapshot) {
|
||||||
|
Ok(protocol::WorkerStateSnapshotApply::Applied) => {
|
||||||
|
self.set_worker_status(self.worker_state.catalog_status());
|
||||||
|
}
|
||||||
|
Ok(
|
||||||
|
protocol::WorkerStateSnapshotApply::Duplicate
|
||||||
|
| protocol::WorkerStateSnapshotApply::Stale,
|
||||||
|
) => {}
|
||||||
|
Err(error) => self.handle_error(
|
||||||
|
ErrorCode::Internal,
|
||||||
|
format!("worker state stream rejected: {error}"),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn handle_worker_event(&mut self, event: Event) -> Option<Method> {
|
pub fn handle_worker_event(&mut self, event: Event) -> Option<Method> {
|
||||||
if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) {
|
if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) {
|
||||||
return None;
|
return None;
|
||||||
@@ -1150,18 +1180,14 @@ impl App {
|
|||||||
self.assistant_streaming = false;
|
self.assistant_streaming = false;
|
||||||
}
|
}
|
||||||
Event::TurnStart { .. } => {
|
Event::TurnStart { .. } => {
|
||||||
self.set_worker_status(WorkerStatus::Running);
|
|
||||||
self.run_requests += 1;
|
self.run_requests += 1;
|
||||||
self.current_tool = None;
|
self.current_tool = None;
|
||||||
self.latest_llm_wait_event = None;
|
self.latest_llm_wait_event = None;
|
||||||
self.assistant_streaming = false;
|
self.assistant_streaming = false;
|
||||||
}
|
}
|
||||||
Event::InvokeStart { .. } => {
|
Event::InvokeStart { .. } => {}
|
||||||
self.set_worker_status(WorkerStatus::Running);
|
|
||||||
}
|
|
||||||
// UI consumers of per-attempt LlmCall semantics remain out of scope;
|
// UI consumers of per-attempt LlmCall semantics remain out of scope;
|
||||||
// the run-level status starts at InvokeStart and TurnStart counts each
|
// authoritative run state comes only from WorkerStateSnapshot.
|
||||||
// LLM request within that run.
|
|
||||||
Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => {
|
Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => {
|
||||||
self.latest_llm_wait_event = None;
|
self.latest_llm_wait_event = None;
|
||||||
}
|
}
|
||||||
@@ -1368,12 +1394,7 @@ impl App {
|
|||||||
output_tokens: self.run_output_tokens,
|
output_tokens: self.run_output_tokens,
|
||||||
});
|
});
|
||||||
self.pending_submit_rollback = None;
|
self.pending_submit_rollback = None;
|
||||||
self.reset_run_state(match result {
|
self.reset_run_state();
|
||||||
RunResult::Paused => WorkerStatus::Paused,
|
|
||||||
RunResult::Finished | RunResult::LimitReached | RunResult::RolledBack => {
|
|
||||||
WorkerStatus::Idle
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Event::CompactStart { .. } => {
|
Event::CompactStart { .. } => {
|
||||||
@@ -1443,7 +1464,7 @@ impl App {
|
|||||||
Event::Snapshot {
|
Event::Snapshot {
|
||||||
session,
|
session,
|
||||||
greeting,
|
greeting,
|
||||||
status,
|
state,
|
||||||
in_flight,
|
in_flight,
|
||||||
internal_workers,
|
internal_workers,
|
||||||
} => {
|
} => {
|
||||||
@@ -1451,7 +1472,7 @@ impl App {
|
|||||||
self.pending_submissions = session.pending_submissions.clone();
|
self.pending_submissions = session.pending_submissions.clone();
|
||||||
self.restore_snapshot(&session, greeting, in_flight);
|
self.restore_snapshot(&session, greeting, in_flight);
|
||||||
self.replace_internal_worker_snapshots(internal_workers);
|
self.replace_internal_worker_snapshots(internal_workers);
|
||||||
self.set_worker_status(status);
|
self.apply_worker_state_snapshot(&state);
|
||||||
}
|
}
|
||||||
Event::InternalWorker {
|
Event::InternalWorker {
|
||||||
worker,
|
worker,
|
||||||
@@ -1461,9 +1482,12 @@ impl App {
|
|||||||
Event::InternalWorkerRemoved { worker, revision } => {
|
Event::InternalWorkerRemoved { worker, revision } => {
|
||||||
self.remove_internal_worker(worker, revision)
|
self.remove_internal_worker(worker, revision)
|
||||||
}
|
}
|
||||||
Event::Status { status } => {
|
Event::WorkerState { snapshot } => {
|
||||||
self.rewind_refresh_fence = false;
|
self.rewind_refresh_fence = false;
|
||||||
self.set_worker_status(status);
|
self.apply_worker_state_snapshot(&snapshot);
|
||||||
|
}
|
||||||
|
Event::CommandAcknowledged { acknowledgement } => {
|
||||||
|
self.apply_worker_state_snapshot(&acknowledgement.state);
|
||||||
}
|
}
|
||||||
// Command telemetry is an operational Web Console surface. The
|
// Command telemetry is an operational Web Console surface. The
|
||||||
// TUI continues to render the final Bash ToolResult from history.
|
// TUI continues to render the final Bash ToolResult from history.
|
||||||
@@ -1503,7 +1527,7 @@ impl App {
|
|||||||
};
|
};
|
||||||
self.completion = None;
|
self.completion = None;
|
||||||
self.close_rewind_picker();
|
self.close_rewind_picker();
|
||||||
self.reset_run_state(self.worker_status);
|
self.reset_run_state();
|
||||||
let mut message = if restored_composer {
|
let mut message = if restored_composer {
|
||||||
format!(
|
format!(
|
||||||
"Rewound session: discarded {} log entries; restored selected input to composer.",
|
"Rewound session: discarded {} log entries; restored selected input to composer.",
|
||||||
@@ -1551,8 +1575,7 @@ impl App {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reset_run_state(&mut self, status: WorkerStatus) {
|
fn reset_run_state(&mut self) {
|
||||||
self.set_worker_status(status);
|
|
||||||
self.run_requests = 0;
|
self.run_requests = 0;
|
||||||
self.run_upload_tokens = 0;
|
self.run_upload_tokens = 0;
|
||||||
self.run_output_tokens = 0;
|
self.run_output_tokens = 0;
|
||||||
@@ -1582,7 +1605,7 @@ impl App {
|
|||||||
"Rolled back empty assistant turn; no local submitted input was available to restore."
|
"Rolled back empty assistant turn; no local submitted input was available to restore."
|
||||||
.to_owned()
|
.to_owned()
|
||||||
};
|
};
|
||||||
self.reset_run_state(WorkerStatus::Idle);
|
self.reset_run_state();
|
||||||
self.blocks.push(Block::Alert {
|
self.blocks.push(Block::Alert {
|
||||||
level: AlertLevel::Warn,
|
level: AlertLevel::Warn,
|
||||||
source: AlertSource::Worker,
|
source: AlertSource::Worker,
|
||||||
@@ -2026,12 +2049,18 @@ impl App {
|
|||||||
self.input_mode = CommandInputMode::Composer;
|
self.input_mode = CommandInputMode::Composer;
|
||||||
self.command_completion_selected = None;
|
self.command_completion_selected = None;
|
||||||
}
|
}
|
||||||
if let Some(Method::ListRewindTargets) = result.method.as_ref() {
|
let mut method = result.method;
|
||||||
|
if let Some(Method::Compact { .. }) = method {
|
||||||
|
method = Some(Method::Compact {
|
||||||
|
command: self.next_command_envelope(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(Method::ListRewindTargets) = method.as_ref() {
|
||||||
self.completion = None;
|
self.completion = None;
|
||||||
self.rewind_picker = None;
|
self.rewind_picker = None;
|
||||||
self.rewind_request_pending = true;
|
self.rewind_request_pending = true;
|
||||||
}
|
}
|
||||||
result.method
|
method
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_command_diagnostic(&mut self, message: impl Into<String>) {
|
fn push_command_diagnostic(&mut self, message: impl Into<String>) {
|
||||||
@@ -2761,8 +2790,8 @@ mod rewind_refresh_tests {
|
|||||||
});
|
});
|
||||||
assert!(!blocks_contain(&app, "stale tail after rewind"));
|
assert!(!blocks_contain(&app, "stale tail after rewind"));
|
||||||
|
|
||||||
app.handle_worker_event(Event::Status {
|
app.handle_worker_event(Event::WorkerState {
|
||||||
status: WorkerStatus::Idle,
|
snapshot: WorkerStatus::Idle.into(),
|
||||||
});
|
});
|
||||||
app.handle_worker_event(Event::TextDelta {
|
app.handle_worker_event(Event::TextDelta {
|
||||||
text: "new live tail after status".into(),
|
text: "new live tail after status".into(),
|
||||||
@@ -3478,7 +3507,7 @@ mod completion_flow_tests {
|
|||||||
let mut app = App::new("test".into());
|
let mut app = App::new("test".into());
|
||||||
app.set_worker_status(WorkerStatus::Paused);
|
app.set_worker_status(WorkerStatus::Paused);
|
||||||
|
|
||||||
assert!(matches!(app.submit_input(), Some(Method::Resume)));
|
assert!(matches!(app.submit_input(), Some(Method::Resume { .. })));
|
||||||
assert_eq!(app.queued_input_count(), 0);
|
assert_eq!(app.queued_input_count(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3533,7 +3562,7 @@ mod completion_flow_tests {
|
|||||||
app.handle_worker_event(Event::Snapshot {
|
app.handle_worker_event(Event::Snapshot {
|
||||||
greeting: test_greeting(),
|
greeting: test_greeting(),
|
||||||
session: public_session(vec![session_start_value]),
|
session: public_session(vec![session_start_value]),
|
||||||
status: WorkerStatus::Running,
|
state: test_worker_state(WorkerStatus::Running),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
});
|
});
|
||||||
@@ -3544,6 +3573,90 @@ mod completion_flow_tests {
|
|||||||
assert!(matches!(app.blocks.first(), Some(Block::Greeting(_))));
|
assert!(matches!(app.blocks.first(), Some(Block::Greeting(_))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn occurrence_events_do_not_infer_foreground_worker_state() {
|
||||||
|
let mut app = App::new("test".into());
|
||||||
|
app.handle_worker_event(Event::TurnStart { turn: 1 });
|
||||||
|
app.handle_worker_event(Event::InvokeStart {
|
||||||
|
kind: protocol::InvokeKind::UserSend,
|
||||||
|
});
|
||||||
|
app.handle_worker_event(Event::RunEnd {
|
||||||
|
result: RunResult::Paused,
|
||||||
|
});
|
||||||
|
assert_eq!(app.worker_state.state, protocol::WorkerState::Idle);
|
||||||
|
assert_eq!(app.worker_status, WorkerStatus::Idle);
|
||||||
|
|
||||||
|
let running = WorkerStateSnapshot {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 1,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 0,
|
||||||
|
};
|
||||||
|
app.handle_worker_event(Event::WorkerState {
|
||||||
|
snapshot: running.clone(),
|
||||||
|
});
|
||||||
|
app.handle_worker_event(Event::RunEnd {
|
||||||
|
result: RunResult::Finished,
|
||||||
|
});
|
||||||
|
assert_eq!(app.worker_state, running);
|
||||||
|
assert_eq!(app.worker_status, WorkerStatus::Running);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_state_events_and_acknowledgements_share_monotonic_application() {
|
||||||
|
let mut app = App::new("test".into());
|
||||||
|
let running = WorkerStateSnapshot {
|
||||||
|
execution_generation: 4,
|
||||||
|
revision: 3,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 2,
|
||||||
|
};
|
||||||
|
app.handle_worker_event(Event::WorkerState {
|
||||||
|
snapshot: running.clone(),
|
||||||
|
});
|
||||||
|
app.handle_worker_event(Event::WorkerState {
|
||||||
|
snapshot: WorkerStateSnapshot {
|
||||||
|
revision: 2,
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..running.clone()
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert_eq!(app.worker_state, running);
|
||||||
|
|
||||||
|
let paused = WorkerStateSnapshot {
|
||||||
|
revision: 4,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Paused,
|
||||||
|
)),
|
||||||
|
last_command_id: 3,
|
||||||
|
..running.clone()
|
||||||
|
};
|
||||||
|
app.handle_worker_event(Event::CommandAcknowledged {
|
||||||
|
acknowledgement: protocol::WorkerCommandAcknowledgement {
|
||||||
|
command_id: 3,
|
||||||
|
command: protocol::WorkerCommandKind::Pause,
|
||||||
|
disposition: protocol::WorkerCommandDisposition::Accepted,
|
||||||
|
state: paused.clone(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert_eq!(app.worker_state, paused);
|
||||||
|
|
||||||
|
app.handle_worker_event(Event::WorkerState {
|
||||||
|
snapshot: WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..paused.clone()
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert_eq!(app.worker_state, paused);
|
||||||
|
assert!(app.run_error_messages.iter().any(|message| {
|
||||||
|
message.contains("conflicting worker state snapshots at generation 4 revision 4")
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn snapshot_replaces_live_error_with_one_durable_run_error_block() {
|
fn snapshot_replaces_live_error_with_one_durable_run_error_block() {
|
||||||
let mut app = App::new("test".into());
|
let mut app = App::new("test".into());
|
||||||
@@ -3551,8 +3664,8 @@ mod completion_flow_tests {
|
|||||||
code: ErrorCode::ProviderError,
|
code: ErrorCode::ProviderError,
|
||||||
message: "provider unavailable".into(),
|
message: "provider unavailable".into(),
|
||||||
});
|
});
|
||||||
app.handle_worker_event(Event::Status {
|
app.handle_worker_event(Event::WorkerState {
|
||||||
status: WorkerStatus::Idle,
|
snapshot: WorkerStatus::Idle.into(),
|
||||||
});
|
});
|
||||||
|
|
||||||
let live_errors = app
|
let live_errors = app
|
||||||
@@ -3577,7 +3690,7 @@ mod completion_flow_tests {
|
|||||||
app.handle_worker_event(Event::Snapshot {
|
app.handle_worker_event(Event::Snapshot {
|
||||||
greeting: test_greeting(),
|
greeting: test_greeting(),
|
||||||
session: public_session(vec![serde_json::to_value(run_errored).unwrap()]),
|
session: public_session(vec![serde_json::to_value(run_errored).unwrap()]),
|
||||||
status: WorkerStatus::Idle,
|
state: test_worker_state(WorkerStatus::Idle),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
});
|
});
|
||||||
@@ -3641,7 +3754,7 @@ mod completion_flow_tests {
|
|||||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Running,
|
state: test_worker_state(WorkerStatus::Running),
|
||||||
in_flight: InFlightSnapshot {
|
in_flight: InFlightSnapshot {
|
||||||
blocks: vec![
|
blocks: vec![
|
||||||
InFlightBlock::Thinking {
|
InFlightBlock::Thinking {
|
||||||
@@ -3968,7 +4081,7 @@ mod completion_flow_tests {
|
|||||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: test_worker_state(WorkerStatus::Idle),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
});
|
});
|
||||||
@@ -4020,7 +4133,7 @@ mod completion_flow_tests {
|
|||||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: test_worker_state(WorkerStatus::Idle),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: vec![InternalWorkerSnapshot {
|
internal_workers: vec![InternalWorkerSnapshot {
|
||||||
worker: InternalWorkerRef {
|
worker: InternalWorkerRef {
|
||||||
@@ -4168,6 +4281,13 @@ mod completion_flow_tests {
|
|||||||
.count()
|
.count()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn test_worker_state(status: WorkerStatus) -> WorkerStateSnapshot {
|
||||||
|
let mut snapshot = WorkerStateSnapshot::from(status);
|
||||||
|
snapshot.execution_generation = 1;
|
||||||
|
snapshot.revision = 1;
|
||||||
|
snapshot
|
||||||
|
}
|
||||||
|
|
||||||
fn test_greeting() -> protocol::Greeting {
|
fn test_greeting() -> protocol::Greeting {
|
||||||
protocol::Greeting {
|
protocol::Greeting {
|
||||||
worker_name: "test".into(),
|
worker_name: "test".into(),
|
||||||
@@ -4194,7 +4314,7 @@ mod completion_flow_tests {
|
|||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
},
|
},
|
||||||
greeting,
|
greeting,
|
||||||
status: WorkerStatus::Idle,
|
state: test_worker_state(WorkerStatus::Idle),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
});
|
});
|
||||||
@@ -4393,7 +4513,7 @@ mod completion_flow_tests {
|
|||||||
app.handle_worker_event(Event::Snapshot {
|
app.handle_worker_event(Event::Snapshot {
|
||||||
greeting: test_greeting(),
|
greeting: test_greeting(),
|
||||||
session: public_session(assistant_item_entries),
|
session: public_session(assistant_item_entries),
|
||||||
status: WorkerStatus::Running,
|
state: test_worker_state(WorkerStatus::Running),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -321,7 +321,7 @@ fn row_line(
|
|||||||
Span::raw(" "),
|
Span::raw(" "),
|
||||||
Span::styled(
|
Span::styled(
|
||||||
pad_column(&worker_state(worker), widths.state),
|
pad_column(&worker_state(worker), widths.state),
|
||||||
state_style(worker.state.as_str()),
|
state_style(worker_state_label(worker)),
|
||||||
),
|
),
|
||||||
Span::raw(" "),
|
Span::raw(" "),
|
||||||
Span::styled(
|
Span::styled(
|
||||||
@@ -341,8 +341,20 @@ fn worker_name(worker: &BackendWorkerSummary) -> &str {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn worker_state_label(worker: &BackendWorkerSummary) -> &str {
|
||||||
|
match worker.worker_state.as_ref().map(|state| &state.state) {
|
||||||
|
Some(protocol::WorkerState::Idle) => "idle",
|
||||||
|
Some(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Paused,
|
||||||
|
))) => "paused",
|
||||||
|
Some(protocol::WorkerState::Busy(_)) => "running",
|
||||||
|
None if worker.state == "stopped" => "stopped",
|
||||||
|
None => "unknown",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn worker_state(worker: &BackendWorkerSummary) -> String {
|
fn worker_state(worker: &BackendWorkerSummary) -> String {
|
||||||
format!("[{}]", worker.state)
|
format!("[{}]", worker_state_label(worker))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn text_width(value: &str) -> usize {
|
fn text_width(value: &str) -> usize {
|
||||||
@@ -413,7 +425,15 @@ mod tests {
|
|||||||
identity: "ws".to_string(),
|
identity: "ws".to_string(),
|
||||||
workspace_id: Some("ws".to_string()),
|
workspace_id: Some("ws".to_string()),
|
||||||
},
|
},
|
||||||
state: "running".to_string(),
|
state: "idle".to_string(),
|
||||||
|
worker_state: Some(protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 1,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 0,
|
||||||
|
}),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: String::new(),
|
retention_state: String::new(),
|
||||||
@@ -450,6 +470,7 @@ mod tests {
|
|||||||
worker.display_name = "Coder".to_string();
|
worker.display_name = "Coder".to_string();
|
||||||
worker.label = "Coder · T-585".to_string();
|
worker.label = "Coder · T-585".to_string();
|
||||||
worker.state = "stopped".to_string();
|
worker.state = "stopped".to_string();
|
||||||
|
worker.worker_state = None;
|
||||||
worker.working_directory = Some(
|
worker.working_directory = Some(
|
||||||
serde_json::from_value(serde_json::json!({
|
serde_json::from_value(serde_json::json!({
|
||||||
"working_directory_id": "001a06a9f0202000000",
|
"working_directory_id": "001a06a9f0202000000",
|
||||||
@@ -478,12 +499,19 @@ mod tests {
|
|||||||
short.label = "Coder".to_string();
|
short.label = "Coder".to_string();
|
||||||
short.display_name = short.label.clone();
|
short.display_name = short.label.clone();
|
||||||
short.state = "idle".to_string();
|
short.state = "idle".to_string();
|
||||||
|
short.worker_state = Some(protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 2,
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
last_command_id: 0,
|
||||||
|
});
|
||||||
|
|
||||||
let mut long = worker("runtime-a", "worker-b", None);
|
let mut long = worker("runtime-a", "worker-b", None);
|
||||||
long.resource_key = "W-100".to_string();
|
long.resource_key = "W-100".to_string();
|
||||||
long.label = "Longer worker · T-9".to_string();
|
long.label = "Longer worker · T-9".to_string();
|
||||||
long.display_name = long.label.clone();
|
long.display_name = long.label.clone();
|
||||||
long.state = "stopped".to_string();
|
long.state = "stopped".to_string();
|
||||||
|
long.worker_state = None;
|
||||||
|
|
||||||
for worker in [&mut short, &mut long] {
|
for worker in [&mut short, &mut long] {
|
||||||
worker.working_directory = Some(
|
worker.working_directory = Some(
|
||||||
|
|||||||
@@ -409,7 +409,12 @@ fn compact_command(invocation: CommandInvocation<'_>) -> CommandExecution {
|
|||||||
let _ = invocation.environment;
|
let _ = invocation.environment;
|
||||||
let _ = invocation.args.raw();
|
let _ = invocation.args.raw();
|
||||||
CommandExecution {
|
CommandExecution {
|
||||||
method: Some(Method::Compact),
|
method: Some(Method::Compact {
|
||||||
|
command: protocol::WorkerCommandEnvelope::for_snapshot(
|
||||||
|
0,
|
||||||
|
&protocol::WorkerStateSnapshot::initial(1),
|
||||||
|
),
|
||||||
|
}),
|
||||||
diagnostics: vec![CommandDiagnostic::new("compact requested")],
|
diagnostics: vec![CommandDiagnostic::new("compact requested")],
|
||||||
exit_command_mode: true,
|
exit_command_mode: true,
|
||||||
clear_input: true,
|
clear_input: true,
|
||||||
@@ -483,7 +488,7 @@ mod tests {
|
|||||||
fn compact_command_returns_compact_method_not_run() {
|
fn compact_command_returns_compact_method_not_run() {
|
||||||
let registry = CommandRegistry::builtins();
|
let registry = CommandRegistry::builtins();
|
||||||
let result = registry.dispatch("compact", &env());
|
let result = registry.dispatch("compact", &env());
|
||||||
assert!(matches!(result.method, Some(Method::Compact)));
|
assert!(matches!(result.method, Some(Method::Compact { .. })));
|
||||||
assert!(result.exit_command_mode);
|
assert!(result.exit_command_mode);
|
||||||
assert!(result.clear_input);
|
assert!(result.clear_input);
|
||||||
assert!(result.diagnostics[0].message.contains("compact requested"));
|
assert!(result.diagnostics[0].message.contains("compact requested"));
|
||||||
|
|||||||
@@ -572,7 +572,7 @@ async fn run_e2e_rewind_fixture(
|
|||||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
entries: Vec::new(),
|
entries: Vec::new(),
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
greeting: Greeting {
|
greeting: Greeting {
|
||||||
worker_name: worker_name.clone(),
|
worker_name: worker_name.clone(),
|
||||||
cwd: workspace_root.display().to_string(),
|
cwd: workspace_root.display().to_string(),
|
||||||
@@ -1438,13 +1438,15 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
|
|||||||
WorkerStatus::Running | WorkerStatus::Paused
|
WorkerStatus::Running | WorkerStatus::Paused
|
||||||
) {
|
) {
|
||||||
app.shutdown_confirm = None;
|
app.shutdown_confirm = None;
|
||||||
return Some(Method::Cancel);
|
let command = app.next_command_envelope();
|
||||||
|
return Some(Method::Cancel { command });
|
||||||
}
|
}
|
||||||
if let Some(pressed_at) = app.shutdown_confirm
|
if let Some(pressed_at) = app.shutdown_confirm
|
||||||
&& pressed_at.elapsed() < CONFIRM_TIMEOUT
|
&& pressed_at.elapsed() < CONFIRM_TIMEOUT
|
||||||
{
|
{
|
||||||
app.shutdown_confirm = None;
|
app.shutdown_confirm = None;
|
||||||
return Some(Method::Shutdown);
|
let command = app.next_command_envelope();
|
||||||
|
return Some(Method::Shutdown { command });
|
||||||
}
|
}
|
||||||
app.shutdown_confirm = Some(std::time::Instant::now());
|
app.shutdown_confirm = Some(std::time::Instant::now());
|
||||||
app.flash_actionbar_notice(
|
app.flash_actionbar_notice(
|
||||||
@@ -1460,7 +1462,8 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option<Method> {
|
|||||||
/// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running).
|
/// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running).
|
||||||
fn handle_pause_or_quit(app: &mut App) -> Option<Method> {
|
fn handle_pause_or_quit(app: &mut App) -> Option<Method> {
|
||||||
if app.worker_status == WorkerStatus::Running {
|
if app.worker_status == WorkerStatus::Running {
|
||||||
return Some(Method::Pause);
|
let command = app.next_command_envelope();
|
||||||
|
return Some(Method::Pause { command });
|
||||||
}
|
}
|
||||||
if let Some(t) = app.quit_confirm
|
if let Some(t) = app.quit_confirm
|
||||||
&& t.elapsed() < CONFIRM_TIMEOUT
|
&& t.elapsed() < CONFIRM_TIMEOUT
|
||||||
@@ -2090,7 +2093,7 @@ mod tests {
|
|||||||
&mut app,
|
&mut app,
|
||||||
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL),
|
KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL),
|
||||||
),
|
),
|
||||||
Some(Method::Pause)
|
Some(Method::Pause { .. })
|
||||||
));
|
));
|
||||||
assert_eq!(app.queued_input_count(), 1);
|
assert_eq!(app.queued_input_count(), 1);
|
||||||
|
|
||||||
@@ -2100,7 +2103,7 @@ mod tests {
|
|||||||
&mut app,
|
&mut app,
|
||||||
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
||||||
),
|
),
|
||||||
Some(Method::Cancel)
|
Some(Method::Cancel { .. })
|
||||||
));
|
));
|
||||||
assert_eq!(app.queued_input_count(), 1);
|
assert_eq!(app.queued_input_count(), 1);
|
||||||
}
|
}
|
||||||
@@ -2114,7 +2117,7 @@ mod tests {
|
|||||||
&mut app,
|
&mut app,
|
||||||
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL),
|
||||||
);
|
);
|
||||||
assert!(matches!(cancel, Some(Method::Cancel)));
|
assert!(matches!(cancel, Some(Method::Cancel { .. })));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2136,7 +2139,7 @@ mod tests {
|
|||||||
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
handle_key(&mut app, ctrl_x()),
|
handle_key(&mut app, ctrl_x()),
|
||||||
Some(Method::Shutdown)
|
Some(Method::Shutdown { .. })
|
||||||
));
|
));
|
||||||
assert!(app.shutdown_confirm.is_none());
|
assert!(app.shutdown_confirm.is_none());
|
||||||
}
|
}
|
||||||
@@ -2466,7 +2469,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let method = handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
let method = handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE));
|
||||||
assert!(matches!(method, Some(protocol::Method::Compact)));
|
assert!(matches!(method, Some(protocol::Method::Compact { .. })));
|
||||||
assert!(!app.is_command_mode());
|
assert!(!app.is_command_mode());
|
||||||
assert_eq!(input_text(&app), "");
|
assert_eq!(input_text(&app), "");
|
||||||
assert_eq!(app.queued_input_count(), 0);
|
assert_eq!(app.queued_input_count(), 0);
|
||||||
@@ -2573,7 +2576,7 @@ mod tests {
|
|||||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
entries: vec![],
|
entries: vec![],
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
});
|
});
|
||||||
@@ -2606,7 +2609,7 @@ mod tests {
|
|||||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
entries: vec![],
|
entries: vec![],
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
});
|
});
|
||||||
@@ -2743,8 +2746,8 @@ mod tests {
|
|||||||
kind: protocol::InternalWorkerKind::SubWorker,
|
kind: protocol::InternalWorkerKind::SubWorker,
|
||||||
},
|
},
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: Box::new(Event::Status {
|
event: Box::new(Event::WorkerState {
|
||||||
status: WorkerStatus::Running,
|
snapshot: WorkerStatus::Running.into(),
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
enter_command_mode(&mut app);
|
enter_command_mode(&mut app);
|
||||||
@@ -2859,8 +2862,8 @@ mod tests {
|
|||||||
kind: protocol::InternalWorkerKind::SubWorker,
|
kind: protocol::InternalWorkerKind::SubWorker,
|
||||||
},
|
},
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: Box::new(Event::Status {
|
event: Box::new(Event::WorkerState {
|
||||||
status: WorkerStatus::Running,
|
snapshot: WorkerStatus::Running.into(),
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -2885,8 +2888,8 @@ mod tests {
|
|||||||
kind: protocol::InternalWorkerKind::SubWorker,
|
kind: protocol::InternalWorkerKind::SubWorker,
|
||||||
},
|
},
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: Box::new(Event::Status {
|
event: Box::new(Event::WorkerState {
|
||||||
status: WorkerStatus::Running,
|
snapshot: WorkerStatus::Running.into(),
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
handle_key(&mut app, key(KeyCode::Tab));
|
handle_key(&mut app, key(KeyCode::Tab));
|
||||||
@@ -2902,7 +2905,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
assert!(first.is_none());
|
assert!(first.is_none());
|
||||||
assert!(matches!(second, Some(Method::Shutdown)));
|
assert!(matches!(second, Some(Method::Shutdown { .. })));
|
||||||
assert_eq!(app.worker_status, WorkerStatus::Idle);
|
assert_eq!(app.worker_status, WorkerStatus::Idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2924,8 +2927,8 @@ mod tests {
|
|||||||
kind: protocol::InternalWorkerKind::SubWorker,
|
kind: protocol::InternalWorkerKind::SubWorker,
|
||||||
},
|
},
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: Box::new(Event::Status {
|
event: Box::new(Event::WorkerState {
|
||||||
status: WorkerStatus::Running,
|
snapshot: WorkerStatus::Running.into(),
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -307,6 +307,8 @@ pub struct WorkerSummary {
|
|||||||
pub worker_id: WorkerId,
|
pub worker_id: WorkerId,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub workspace_id: Option<String>,
|
pub workspace_id: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||||
@@ -325,6 +327,8 @@ pub struct WorkerDetail {
|
|||||||
pub worker_id: WorkerId,
|
pub worker_id: WorkerId,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub workspace_id: Option<String>,
|
pub workspace_id: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||||
@@ -341,6 +345,8 @@ pub struct WorkerDetail {
|
|||||||
pub struct WorkerLifecycleAck {
|
pub struct WorkerLifecycleAck {
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -15,18 +15,6 @@ use std::fmt;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use workdir::WorkdirSessionHandle;
|
use workdir::WorkdirSessionHandle;
|
||||||
|
|
||||||
/// Current execution-side run state for a Worker.
|
|
||||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum WorkerExecutionRunState {
|
|
||||||
#[default]
|
|
||||||
Stopped,
|
|
||||||
Idle,
|
|
||||||
Busy,
|
|
||||||
Rejected,
|
|
||||||
Errored,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Execution operation that produced a result.
|
/// Execution operation that produced a result.
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -55,7 +43,8 @@ pub struct WorkerSubmissionAck {
|
|||||||
pub struct WorkerExecutionResult {
|
pub struct WorkerExecutionResult {
|
||||||
pub operation: WorkerExecutionOperation,
|
pub operation: WorkerExecutionOperation,
|
||||||
pub outcome: WorkerExecutionOutcome,
|
pub outcome: WorkerExecutionOutcome,
|
||||||
pub run_state: WorkerExecutionRunState,
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub message: Option<String>,
|
pub message: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
@@ -74,22 +63,23 @@ pub enum WorkerExecutionOutcome {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerExecutionResult {
|
impl WorkerExecutionResult {
|
||||||
pub fn accepted(
|
pub fn accepted(operation: WorkerExecutionOperation) -> Self {
|
||||||
operation: WorkerExecutionOperation,
|
|
||||||
run_state: WorkerExecutionRunState,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
Self {
|
||||||
operation,
|
operation,
|
||||||
outcome: WorkerExecutionOutcome::Accepted,
|
outcome: WorkerExecutionOutcome::Accepted,
|
||||||
run_state,
|
worker_state: None,
|
||||||
message: None,
|
message: None,
|
||||||
submission: None,
|
submission: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn with_worker_state(mut self, worker_state: protocol::WorkerStateSnapshot) -> Self {
|
||||||
|
self.worker_state = Some(worker_state);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn accepted_submission(
|
pub fn accepted_submission(
|
||||||
operation: WorkerExecutionOperation,
|
operation: WorkerExecutionOperation,
|
||||||
run_state: WorkerExecutionRunState,
|
|
||||||
submission_request_id: impl Into<String>,
|
submission_request_id: impl Into<String>,
|
||||||
submission_id: impl Into<String>,
|
submission_id: impl Into<String>,
|
||||||
disposition: protocol::SubmissionDisposition,
|
disposition: protocol::SubmissionDisposition,
|
||||||
@@ -97,7 +87,7 @@ impl WorkerExecutionResult {
|
|||||||
Self {
|
Self {
|
||||||
operation,
|
operation,
|
||||||
outcome: WorkerExecutionOutcome::Accepted,
|
outcome: WorkerExecutionOutcome::Accepted,
|
||||||
run_state,
|
worker_state: None,
|
||||||
message: None,
|
message: None,
|
||||||
submission: Some(WorkerSubmissionAck {
|
submission: Some(WorkerSubmissionAck {
|
||||||
submission_request_id: submission_request_id.into(),
|
submission_request_id: submission_request_id.into(),
|
||||||
@@ -111,7 +101,7 @@ impl WorkerExecutionResult {
|
|||||||
Self {
|
Self {
|
||||||
operation,
|
operation,
|
||||||
outcome: WorkerExecutionOutcome::Busy,
|
outcome: WorkerExecutionOutcome::Busy,
|
||||||
run_state: WorkerExecutionRunState::Busy,
|
worker_state: None,
|
||||||
message: Some(message.into()),
|
message: Some(message.into()),
|
||||||
submission: None,
|
submission: None,
|
||||||
}
|
}
|
||||||
@@ -121,7 +111,7 @@ impl WorkerExecutionResult {
|
|||||||
Self {
|
Self {
|
||||||
operation,
|
operation,
|
||||||
outcome: WorkerExecutionOutcome::Rejected,
|
outcome: WorkerExecutionOutcome::Rejected,
|
||||||
run_state: WorkerExecutionRunState::Stopped,
|
worker_state: None,
|
||||||
message: Some(message.into()),
|
message: Some(message.into()),
|
||||||
submission: None,
|
submission: None,
|
||||||
}
|
}
|
||||||
@@ -131,7 +121,7 @@ impl WorkerExecutionResult {
|
|||||||
Self {
|
Self {
|
||||||
operation,
|
operation,
|
||||||
outcome: WorkerExecutionOutcome::Errored,
|
outcome: WorkerExecutionOutcome::Errored,
|
||||||
run_state: WorkerExecutionRunState::Errored,
|
worker_state: None,
|
||||||
message: Some(message.into()),
|
message: Some(message.into()),
|
||||||
submission: None,
|
submission: None,
|
||||||
}
|
}
|
||||||
@@ -141,7 +131,7 @@ impl WorkerExecutionResult {
|
|||||||
Self {
|
Self {
|
||||||
operation,
|
operation,
|
||||||
outcome: WorkerExecutionOutcome::Unsupported,
|
outcome: WorkerExecutionOutcome::Unsupported,
|
||||||
run_state: WorkerExecutionRunState::Stopped,
|
worker_state: None,
|
||||||
message: Some(message.into()),
|
message: Some(message.into()),
|
||||||
submission: None,
|
submission: None,
|
||||||
}
|
}
|
||||||
@@ -280,7 +270,6 @@ pub struct WorkerExecutionRestoreRequest {
|
|||||||
pub enum WorkerExecutionSpawnResult {
|
pub enum WorkerExecutionSpawnResult {
|
||||||
Connected {
|
Connected {
|
||||||
handle: WorkerExecutionHandle,
|
handle: WorkerExecutionHandle,
|
||||||
run_state: WorkerExecutionRunState,
|
|
||||||
working_directory: Option<WorkingDirectoryStatus>,
|
working_directory: Option<WorkingDirectoryStatus>,
|
||||||
},
|
},
|
||||||
Rejected(WorkerExecutionResult),
|
Rejected(WorkerExecutionResult),
|
||||||
@@ -290,12 +279,10 @@ pub enum WorkerExecutionSpawnResult {
|
|||||||
impl WorkerExecutionSpawnResult {
|
impl WorkerExecutionSpawnResult {
|
||||||
pub fn connected(
|
pub fn connected(
|
||||||
handle: WorkerExecutionHandle,
|
handle: WorkerExecutionHandle,
|
||||||
run_state: WorkerExecutionRunState,
|
|
||||||
working_directory: Option<WorkingDirectoryStatus>,
|
working_directory: Option<WorkingDirectoryStatus>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self::Connected {
|
Self::Connected {
|
||||||
handle,
|
handle,
|
||||||
run_state,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -623,7 +610,6 @@ mod tests {
|
|||||||
fn submission_ack_survives_json_round_trip() {
|
fn submission_ack_survives_json_round_trip() {
|
||||||
let result = WorkerExecutionResult::accepted_submission(
|
let result = WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Busy,
|
|
||||||
"request-1",
|
"request-1",
|
||||||
"submission-1",
|
"submission-1",
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
|
|||||||
@@ -2205,8 +2205,8 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use crate::execution::{
|
use crate::execution::{
|
||||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
|
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest,
|
||||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
WorkerExecutionSpawnResult,
|
||||||
};
|
};
|
||||||
use crate::management::RuntimeOptions;
|
use crate::management::RuntimeOptions;
|
||||||
use axum::body::to_bytes;
|
use axum::body::to_bytes;
|
||||||
@@ -2895,7 +2895,6 @@ mod tests {
|
|||||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: request
|
working_directory: request
|
||||||
.working_directory
|
.working_directory
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -2909,7 +2908,6 @@ mod tests {
|
|||||||
) -> WorkerExecutionSpawnResult {
|
) -> WorkerExecutionSpawnResult {
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: request.previous_working_directory,
|
working_directory: request.previous_working_directory,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2922,24 +2920,17 @@ mod tests {
|
|||||||
if let Some(submission_id) = input.submission_request_id {
|
if let Some(submission_id) = input.submission_request_id {
|
||||||
WorkerExecutionResult::accepted_submission(
|
WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
submission_id.clone(),
|
submission_id.clone(),
|
||||||
submission_id,
|
submission_id,
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
|
||||||
WorkerExecutionOperation::Input,
|
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
|
||||||
WorkerExecutionOperation::Stop,
|
|
||||||
WorkerExecutionRunState::Stopped,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3211,8 +3202,7 @@ mod ws_tests {
|
|||||||
};
|
};
|
||||||
use crate::execution::{
|
use crate::execution::{
|
||||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||||
WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest,
|
WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||||
WorkerExecutionSpawnResult,
|
|
||||||
};
|
};
|
||||||
use crate::management::RuntimeOptions;
|
use crate::management::RuntimeOptions;
|
||||||
use futures::{SinkExt, StreamExt};
|
use futures::{SinkExt, StreamExt};
|
||||||
@@ -3232,7 +3222,6 @@ mod ws_tests {
|
|||||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: request
|
working_directory: request
|
||||||
.working_directory
|
.working_directory
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -3248,28 +3237,46 @@ mod ws_tests {
|
|||||||
if let Some(submission_id) = input.submission_request_id {
|
if let Some(submission_id) = input.submission_request_id {
|
||||||
WorkerExecutionResult::accepted_submission(
|
WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
submission_id.clone(),
|
submission_id.clone(),
|
||||||
submission_id,
|
submission_id,
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
|
||||||
WorkerExecutionOperation::Input,
|
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option<protocol::Event> {
|
||||||
|
Some(protocol::Event::Snapshot {
|
||||||
|
session: protocol::SessionSnapshot {
|
||||||
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
|
entries: Vec::new(),
|
||||||
|
},
|
||||||
|
greeting: protocol::Greeting {
|
||||||
|
worker_name: handle.worker_ref().worker_id.to_string(),
|
||||||
|
cwd: String::new(),
|
||||||
|
provider: "ws-test".to_string(),
|
||||||
|
model: "ws-test".to_string(),
|
||||||
|
scope_summary: "WebSocket test execution snapshot".to_string(),
|
||||||
|
tools: Vec::new(),
|
||||||
|
context_window: 0,
|
||||||
|
context_tokens: 0,
|
||||||
|
},
|
||||||
|
state: protocol::WorkerStateSnapshot::initial(1),
|
||||||
|
in_flight: protocol::InFlightSnapshot {
|
||||||
|
blocks: Vec::new(),
|
||||||
|
commands: Vec::new(),
|
||||||
|
},
|
||||||
|
internal_workers: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn dispatch_method(
|
fn dispatch_method(
|
||||||
&self,
|
&self,
|
||||||
_handle: &WorkerExecutionHandle,
|
_handle: &WorkerExecutionHandle,
|
||||||
_method: protocol::Method,
|
_method: protocol::Method,
|
||||||
) -> WorkerExecutionResult {
|
) -> WorkerExecutionResult {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::ProtocolMethod)
|
||||||
WorkerExecutionOperation::ProtocolMethod,
|
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3480,16 +3487,16 @@ mod ws_tests {
|
|||||||
runtime
|
runtime
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
&other.worker_ref,
|
&other.worker_ref,
|
||||||
protocol::Event::Status {
|
protocol::Event::WorkerState {
|
||||||
status: protocol::WorkerStatus::Running,
|
snapshot: protocol::WorkerStatus::Running.into(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
protocol::Event::Status {
|
protocol::Event::WorkerState {
|
||||||
status: protocol::WorkerStatus::Running,
|
snapshot: protocol::WorkerStatus::Running.into(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -3504,7 +3511,16 @@ mod ws_tests {
|
|||||||
..
|
..
|
||||||
}) if delivered_subscription_id == subscription_id
|
}) if delivered_subscription_id == subscription_id
|
||||||
&& worker.worker_id.as_str() == worker_ref.worker_id.to_string()
|
&& worker.worker_id.as_str() == worker_ref.worker_id.to_string()
|
||||||
&& worker.state == protocol::subscription::SubscriptionWorkerState::Running
|
&& worker.state == protocol::subscription::SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running
|
||||||
|
)),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
));
|
));
|
||||||
|
|
||||||
let unsubscribe_request_id =
|
let unsubscribe_request_id =
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ use crate::error::RuntimeError;
|
|||||||
use crate::execution::WorkerExecutionRestoreRequest;
|
use crate::execution::WorkerExecutionRestoreRequest;
|
||||||
use crate::execution::{
|
use crate::execution::{
|
||||||
WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle,
|
WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle,
|
||||||
WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionRunState,
|
WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest,
|
||||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
WorkerExecutionSpawnResult,
|
||||||
};
|
};
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
use crate::fs_store::{
|
use crate::fs_store::{
|
||||||
@@ -700,6 +700,7 @@ impl Runtime {
|
|||||||
worker_ref: worker_ref.clone(),
|
worker_ref: worker_ref.clone(),
|
||||||
worker_id: worker_id.clone(),
|
worker_id: worker_id.clone(),
|
||||||
status: WorkerStatus::Stopped,
|
status: WorkerStatus::Stopped,
|
||||||
|
worker_state: None,
|
||||||
workspace_id: scope.map(|scope| scope.workspace_id.clone()),
|
workspace_id: scope.map(|scope| scope.workspace_id.clone()),
|
||||||
request: durable_request,
|
request: durable_request,
|
||||||
run_generation: 1,
|
run_generation: 1,
|
||||||
@@ -725,12 +726,11 @@ impl Runtime {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let spawn_result = backend.spawn_worker(spawn_request);
|
let spawn_result = backend.spawn_worker(spawn_request);
|
||||||
let (handle, run_state, working_directory) = match spawn_result {
|
let (handle, working_directory) = match spawn_result {
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle,
|
handle,
|
||||||
run_state,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
} => (handle, run_state, working_directory),
|
} => (handle, working_directory),
|
||||||
WorkerExecutionSpawnResult::Rejected(result)
|
WorkerExecutionSpawnResult::Rejected(result)
|
||||||
| WorkerExecutionSpawnResult::Errored(result) => {
|
| WorkerExecutionSpawnResult::Errored(result) => {
|
||||||
self.rollback_failed_create(&worker_ref)?;
|
self.rollback_failed_create(&worker_ref)?;
|
||||||
@@ -785,11 +785,9 @@ impl Runtime {
|
|||||||
result,
|
result,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let initial_run_state = dispatch_result.run_state;
|
|
||||||
let detail = self.commit_created_worker(
|
let detail = self.commit_created_worker(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
handle,
|
handle,
|
||||||
initial_run_state,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
dispatch_result,
|
dispatch_result,
|
||||||
)?;
|
)?;
|
||||||
@@ -799,9 +797,8 @@ impl Runtime {
|
|||||||
self.commit_created_worker(
|
self.commit_created_worker(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
handle,
|
handle,
|
||||||
run_state,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn, run_state),
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1086,13 +1083,12 @@ impl Runtime {
|
|||||||
match backend.restore_worker(request) {
|
match backend.restore_worker(request) {
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle,
|
handle,
|
||||||
run_state,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
} => {
|
} => {
|
||||||
self.commit_restored_worker_execution(
|
self.commit_restored_worker_execution(
|
||||||
worker_ref,
|
worker_ref,
|
||||||
handle,
|
handle,
|
||||||
run_state,
|
WorkerStatus::Idle,
|
||||||
working_directory,
|
working_directory,
|
||||||
)?;
|
)?;
|
||||||
self.worker_detail(worker_ref)
|
self.worker_detail(worker_ref)
|
||||||
@@ -1222,7 +1218,9 @@ impl Runtime {
|
|||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.status = worker_status_from_run_state(dispatch_result.run_state);
|
if let Some(snapshot) = dispatch_result.worker_state.as_ref() {
|
||||||
|
let _ = worker.apply_worker_state(snapshot);
|
||||||
|
}
|
||||||
let status = worker.status;
|
let status = worker.status;
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
if let Some(payload) = input_protocol_event(&input) {
|
if let Some(payload) = input_protocol_event(&input) {
|
||||||
@@ -1431,7 +1429,7 @@ impl Runtime {
|
|||||||
let entries = self.worker_completions(worker_ref, kind, &prefix)?;
|
let entries = self.worker_completions(worker_ref, kind, &prefix)?;
|
||||||
return Ok(vec![Event::Completions { kind, entries }]);
|
return Ok(vec![Event::Completions { kind, entries }]);
|
||||||
}
|
}
|
||||||
if matches!(&method, Method::Shutdown) {
|
if matches!(&method, Method::Shutdown { .. }) {
|
||||||
self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?;
|
self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?;
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
@@ -1481,16 +1479,19 @@ impl Runtime {
|
|||||||
&self,
|
&self,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
handle: WorkerExecutionHandle,
|
handle: WorkerExecutionHandle,
|
||||||
run_state: WorkerExecutionRunState,
|
|
||||||
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
||||||
_result: WorkerExecutionResult,
|
result: WorkerExecutionResult,
|
||||||
) -> Result<WorkerDetail, RuntimeError> {
|
) -> Result<WorkerDetail, RuntimeError> {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
let detail = {
|
let detail = {
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.execution_handle = Some(handle);
|
worker.execution_handle = Some(handle);
|
||||||
worker.execution_bound = true;
|
worker.execution_bound = true;
|
||||||
worker.status = worker_status_from_run_state(run_state);
|
worker.status = WorkerStatus::Idle;
|
||||||
|
worker.worker_state = None;
|
||||||
|
if let Some(snapshot) = result.worker_state.as_ref() {
|
||||||
|
let _ = worker.apply_worker_state(snapshot);
|
||||||
|
}
|
||||||
worker.restore_intent = restore_intent_for_status(worker.status);
|
worker.restore_intent = restore_intent_for_status(worker.status);
|
||||||
worker.working_directory = working_directory;
|
worker.working_directory = working_directory;
|
||||||
worker.detail()
|
worker.detail()
|
||||||
@@ -1518,16 +1519,26 @@ impl Runtime {
|
|||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
result: WorkerExecutionResult,
|
result: WorkerExecutionResult,
|
||||||
) -> Result<(), RuntimeError> {
|
) -> Result<(), RuntimeError> {
|
||||||
|
// Accepted dispatch without a state snapshot is transport evidence only;
|
||||||
|
// the revisioned protocol stream remains live authority. Test/detached
|
||||||
|
// backends may return an exact full snapshot as their acknowledgement.
|
||||||
|
if !result.is_accepted() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let Some(snapshot) = result.worker_state else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
if result.is_accepted() {
|
|
||||||
let status = worker_status_from_run_state(result.run_state);
|
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.status = status;
|
let applied = worker
|
||||||
worker.restore_intent = restore_intent_for_status(status);
|
.apply_worker_state(&snapshot)
|
||||||
|
.is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied));
|
||||||
|
if !applied {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||||
state.persist_runtime_snapshot()?;
|
state.persist_runtime_snapshot()?;
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
state.persist_worker(&worker_ref.worker_id)?;
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1615,20 +1626,26 @@ impl Runtime {
|
|||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
reason: Option<String>,
|
reason: Option<String>,
|
||||||
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
||||||
let current = {
|
{
|
||||||
let state = self.lock()?;
|
let state = self.lock()?;
|
||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
state.worker(worker_ref)?.status
|
if state.worker(worker_ref)?.status == WorkerStatus::Stopped {
|
||||||
};
|
|
||||||
if matches!(current, WorkerStatus::Idle | WorkerStatus::Stopped) {
|
|
||||||
return Ok(WorkerLifecycleAck {
|
return Ok(WorkerLifecycleAck {
|
||||||
worker_ref: worker_ref.clone(),
|
worker_ref: worker_ref.clone(),
|
||||||
status: current,
|
status: WorkerStatus::Stopped,
|
||||||
|
worker_state: None,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
}
|
||||||
self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?;
|
self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?;
|
||||||
let _ = reason;
|
let _ = reason;
|
||||||
self.transition_worker_preserving_execution(worker_ref, WorkerStatus::Idle)
|
let state = self.lock()?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
Ok(WorkerLifecycleAck {
|
||||||
|
worker_ref: worker_ref.clone(),
|
||||||
|
status: worker.status,
|
||||||
|
worker_state: worker.worker_state.clone(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete a non-running Worker through a workspace-scoped Runtime authorization context.
|
/// Delete a non-running Worker through a workspace-scoped Runtime authorization context.
|
||||||
@@ -1715,27 +1732,9 @@ impl Runtime {
|
|||||||
return Ok(snapshot);
|
return Ok(snapshot);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(protocol::Event::Snapshot {
|
Err(RuntimeError::WorkerExecutionUnavailable {
|
||||||
session: protocol::SessionSnapshot {
|
worker_id: worker_ref.worker_id,
|
||||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
message: "authoritative Worker snapshot is unavailable".to_string(),
|
||||||
entries: Vec::new(),
|
|
||||||
},
|
|
||||||
greeting: protocol::Greeting {
|
|
||||||
worker_name: worker_ref.worker_id.to_string(),
|
|
||||||
cwd: String::new(),
|
|
||||||
provider: "worker-runtime".to_string(),
|
|
||||||
model: "worker-runtime".to_string(),
|
|
||||||
scope_summary: "runtime worker observation".to_string(),
|
|
||||||
tools: Vec::new(),
|
|
||||||
context_window: 0,
|
|
||||||
context_tokens: 0,
|
|
||||||
},
|
|
||||||
status: protocol::WorkerStatus::Idle,
|
|
||||||
in_flight: protocol::InFlightSnapshot {
|
|
||||||
blocks: Vec::new(),
|
|
||||||
commands: Vec::new(),
|
|
||||||
},
|
|
||||||
internal_workers: Vec::new(),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1774,12 +1773,13 @@ impl Runtime {
|
|||||||
) -> Result<WorkerObservationEvent, RuntimeError> {
|
) -> Result<WorkerObservationEvent, RuntimeError> {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
state.ensure_worker_ref(worker_ref)?;
|
state.ensure_worker_ref(worker_ref)?;
|
||||||
let status_changed = state.project_protocol_event_to_status(worker_ref, &payload);
|
let worker_state_changed =
|
||||||
|
state.project_protocol_event_to_worker_state(worker_ref, &payload);
|
||||||
let activity_changed = state.project_internal_worker_activity(worker_ref, &payload);
|
let activity_changed = state.project_internal_worker_activity(worker_ref, &payload);
|
||||||
if status_changed || activity_changed {
|
if worker_state_changed || activity_changed {
|
||||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||||
}
|
}
|
||||||
if status_changed {
|
if worker_state_changed {
|
||||||
state.persist_runtime_snapshot()?;
|
state.persist_runtime_snapshot()?;
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
state.persist_worker(&worker_ref.worker_id)?;
|
||||||
}
|
}
|
||||||
@@ -1815,26 +1815,6 @@ impl Runtime {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn transition_worker_preserving_execution(
|
|
||||||
&self,
|
|
||||||
worker_ref: &WorkerRef,
|
|
||||||
status: WorkerStatus,
|
|
||||||
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
|
||||||
let mut state = self.lock()?;
|
|
||||||
state.ensure_running()?;
|
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
|
||||||
worker.status = status;
|
|
||||||
worker.restore_intent = restore_intent_for_status(status);
|
|
||||||
let status = worker.status;
|
|
||||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
|
||||||
state.persist_runtime_snapshot()?;
|
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
|
||||||
Ok(WorkerLifecycleAck {
|
|
||||||
worker_ref: worker_ref.clone(),
|
|
||||||
status,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn transition_worker(
|
fn transition_worker(
|
||||||
&self,
|
&self,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
@@ -1846,6 +1826,7 @@ impl Runtime {
|
|||||||
|
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.status = status;
|
worker.status = status;
|
||||||
|
worker.worker_state = None;
|
||||||
worker.restore_intent = restore_intent_for_status(status);
|
worker.restore_intent = restore_intent_for_status(status);
|
||||||
worker.execution_handle = None;
|
worker.execution_handle = None;
|
||||||
worker.internal_workers.clear();
|
worker.internal_workers.clear();
|
||||||
@@ -1856,6 +1837,7 @@ impl Runtime {
|
|||||||
Ok(WorkerLifecycleAck {
|
Ok(WorkerLifecycleAck {
|
||||||
worker_ref: worker_ref.clone(),
|
worker_ref: worker_ref.clone(),
|
||||||
status,
|
status,
|
||||||
|
worker_state: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1968,12 +1950,11 @@ impl Runtime {
|
|||||||
match backend.restore_worker(request) {
|
match backend.restore_worker(request) {
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle,
|
handle,
|
||||||
run_state,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
} => self.commit_restored_worker_execution(
|
} => self.commit_restored_worker_execution(
|
||||||
&candidate.worker_ref,
|
&candidate.worker_ref,
|
||||||
handle,
|
handle,
|
||||||
run_state,
|
WorkerStatus::Idle,
|
||||||
working_directory,
|
working_directory,
|
||||||
)?,
|
)?,
|
||||||
WorkerExecutionSpawnResult::Rejected(result)
|
WorkerExecutionSpawnResult::Rejected(result)
|
||||||
@@ -1990,7 +1971,7 @@ impl Runtime {
|
|||||||
&self,
|
&self,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
handle: WorkerExecutionHandle,
|
handle: WorkerExecutionHandle,
|
||||||
run_state: WorkerExecutionRunState,
|
status: WorkerStatus,
|
||||||
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
||||||
) -> Result<(), RuntimeError> {
|
) -> Result<(), RuntimeError> {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
@@ -1999,7 +1980,7 @@ impl Runtime {
|
|||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.execution_handle = Some(handle);
|
worker.execution_handle = Some(handle);
|
||||||
worker.execution_bound = true;
|
worker.execution_bound = true;
|
||||||
worker.status = worker_status_from_run_state(run_state);
|
worker.status = status;
|
||||||
worker.restore_intent = restore_intent_for_status(worker.status);
|
worker.restore_intent = restore_intent_for_status(worker.status);
|
||||||
worker.working_directory = working_directory;
|
worker.working_directory = working_directory;
|
||||||
}
|
}
|
||||||
@@ -2281,6 +2262,7 @@ impl RuntimeState {
|
|||||||
worker_ref: worker.worker_ref,
|
worker_ref: worker.worker_ref,
|
||||||
worker_id: worker.worker_id,
|
worker_id: worker.worker_id,
|
||||||
status: worker.status,
|
status: worker.status,
|
||||||
|
worker_state: None,
|
||||||
workspace_id: worker.workspace_id,
|
workspace_id: worker.workspace_id,
|
||||||
request: worker.request,
|
request: worker.request,
|
||||||
run_generation,
|
run_generation,
|
||||||
@@ -2610,6 +2592,7 @@ impl RuntimeState {
|
|||||||
.get(&worker.worker_id)
|
.get(&worker.worker_id)
|
||||||
.copied()
|
.copied()
|
||||||
.unwrap_or(0),
|
.unwrap_or(0),
|
||||||
|
worker_state: worker.worker_state.clone(),
|
||||||
state: subscription_worker_state(worker.status),
|
state: subscription_worker_state(worker.status),
|
||||||
has_running_internal_workers: worker
|
has_running_internal_workers: worker
|
||||||
.internal_workers
|
.internal_workers
|
||||||
@@ -2867,7 +2850,7 @@ impl RuntimeState {
|
|||||||
) {
|
) {
|
||||||
match event {
|
match event {
|
||||||
protocol::Event::Snapshot {
|
protocol::Event::Snapshot {
|
||||||
status,
|
state,
|
||||||
internal_workers,
|
internal_workers,
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
@@ -2875,7 +2858,7 @@ impl RuntimeState {
|
|||||||
statuses.insert(
|
statuses.insert(
|
||||||
worker.session_id.clone(),
|
worker.session_id.clone(),
|
||||||
InternalWorkerActivity {
|
InternalWorkerActivity {
|
||||||
status: *status,
|
status: state.catalog_status(),
|
||||||
parent_session_id: worker.parent_session_id.clone(),
|
parent_session_id: worker.parent_session_id.clone(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -2888,26 +2871,17 @@ impl RuntimeState {
|
|||||||
event,
|
event,
|
||||||
..
|
..
|
||||||
} => Self::project_internal_worker_event(statuses, nested_worker, event),
|
} => Self::project_internal_worker_event(statuses, nested_worker, event),
|
||||||
protocol::Event::Status { status } => {
|
protocol::Event::WorkerState { snapshot }
|
||||||
statuses.insert(
|
| protocol::Event::CommandAcknowledged {
|
||||||
worker.session_id.clone(),
|
acknowledgement:
|
||||||
InternalWorkerActivity {
|
protocol::WorkerCommandAcknowledgement {
|
||||||
status: *status,
|
state: snapshot, ..
|
||||||
parent_session_id: worker.parent_session_id.clone(),
|
|
||||||
},
|
},
|
||||||
);
|
} => {
|
||||||
}
|
|
||||||
protocol::Event::RunEnd { result } => {
|
|
||||||
let status = match result {
|
|
||||||
protocol::RunResult::Paused => protocol::WorkerStatus::Paused,
|
|
||||||
protocol::RunResult::Finished
|
|
||||||
| protocol::RunResult::LimitReached
|
|
||||||
| protocol::RunResult::RolledBack => protocol::WorkerStatus::Idle,
|
|
||||||
};
|
|
||||||
statuses.insert(
|
statuses.insert(
|
||||||
worker.session_id.clone(),
|
worker.session_id.clone(),
|
||||||
InternalWorkerActivity {
|
InternalWorkerActivity {
|
||||||
status,
|
status: snapshot.catalog_status(),
|
||||||
parent_session_id: worker.parent_session_id.clone(),
|
parent_session_id: worker.parent_session_id.clone(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -2954,7 +2928,7 @@ impl RuntimeState {
|
|||||||
Self::update_internal_worker_activity(&mut worker.internal_workers, event)
|
Self::update_internal_worker_activity(&mut worker.internal_workers, event)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn project_protocol_event_to_status(
|
fn project_protocol_event_to_worker_state(
|
||||||
&mut self,
|
&mut self,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
event: &protocol::Event,
|
event: &protocol::Event,
|
||||||
@@ -2962,38 +2936,26 @@ impl RuntimeState {
|
|||||||
let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else {
|
let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let next_status = match event {
|
let incoming = match event {
|
||||||
protocol::Event::Status {
|
protocol::Event::WorkerState { snapshot }
|
||||||
status: protocol::WorkerStatus::Running,
|
| protocol::Event::Snapshot {
|
||||||
} => Some(WorkerStatus::Running),
|
state: snapshot, ..
|
||||||
protocol::Event::Status {
|
|
||||||
status: protocol::WorkerStatus::Idle,
|
|
||||||
} => Some(WorkerStatus::Idle),
|
|
||||||
protocol::Event::Status {
|
|
||||||
status: protocol::WorkerStatus::Paused,
|
|
||||||
} => Some(WorkerStatus::Paused),
|
|
||||||
protocol::Event::Snapshot { status, .. } => match status {
|
|
||||||
protocol::WorkerStatus::Running => Some(WorkerStatus::Running),
|
|
||||||
protocol::WorkerStatus::Idle => Some(WorkerStatus::Idle),
|
|
||||||
protocol::WorkerStatus::Paused => Some(WorkerStatus::Paused),
|
|
||||||
protocol::WorkerStatus::Stopped => Some(WorkerStatus::Stopped),
|
|
||||||
},
|
|
||||||
protocol::Event::RunEnd { result } => match result {
|
|
||||||
protocol::RunResult::Finished | protocol::RunResult::RolledBack => {
|
|
||||||
Some(WorkerStatus::Idle)
|
|
||||||
}
|
}
|
||||||
protocol::RunResult::Paused => Some(WorkerStatus::Paused),
|
| protocol::Event::CommandAcknowledged {
|
||||||
protocol::RunResult::LimitReached => Some(WorkerStatus::Idle),
|
acknowledgement:
|
||||||
|
protocol::WorkerCommandAcknowledgement {
|
||||||
|
state: snapshot, ..
|
||||||
},
|
},
|
||||||
_ => None,
|
} => snapshot,
|
||||||
|
_ => return false,
|
||||||
};
|
};
|
||||||
if let Some(next_status) = next_status {
|
match worker.apply_worker_state(incoming) {
|
||||||
let changed = worker.status != next_status;
|
Ok(protocol::WorkerStateSnapshotApply::Applied) => true,
|
||||||
worker.status = next_status;
|
Ok(
|
||||||
worker.restore_intent = restore_intent_for_status(next_status);
|
protocol::WorkerStateSnapshotApply::Duplicate
|
||||||
changed
|
| protocol::WorkerStateSnapshotApply::Stale,
|
||||||
} else {
|
)
|
||||||
false
|
| Err(_) => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3009,6 +2971,7 @@ struct WorkerRecord {
|
|||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
worker_id: WorkerId,
|
worker_id: WorkerId,
|
||||||
status: WorkerStatus,
|
status: WorkerStatus,
|
||||||
|
worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
workspace_id: Option<String>,
|
workspace_id: Option<String>,
|
||||||
request: CreateWorkerRequest,
|
request: CreateWorkerRequest,
|
||||||
run_generation: u64,
|
run_generation: u64,
|
||||||
@@ -3020,6 +2983,19 @@ struct WorkerRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerRecord {
|
impl WorkerRecord {
|
||||||
|
fn apply_worker_state(
|
||||||
|
&mut self,
|
||||||
|
incoming: &protocol::WorkerStateSnapshot,
|
||||||
|
) -> Result<protocol::WorkerStateSnapshotApply, protocol::WorkerStateSnapshotConflict> {
|
||||||
|
match self.worker_state.as_mut() {
|
||||||
|
Some(current) => protocol::apply_worker_state_snapshot(current, incoming),
|
||||||
|
None => {
|
||||||
|
self.worker_state = Some(incoming.clone());
|
||||||
|
Ok(protocol::WorkerStateSnapshotApply::Applied)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
|
fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
|
||||||
self.workspace_id.as_deref() == Some(workspace_id)
|
self.workspace_id.as_deref() == Some(workspace_id)
|
||||||
}
|
}
|
||||||
@@ -3029,6 +3005,7 @@ impl WorkerRecord {
|
|||||||
worker_ref: self.worker_ref.clone(),
|
worker_ref: self.worker_ref.clone(),
|
||||||
worker_id: self.worker_id,
|
worker_id: self.worker_id,
|
||||||
status: self.status,
|
status: self.status,
|
||||||
|
worker_state: self.worker_state.clone(),
|
||||||
workspace_id: self.workspace_id.clone(),
|
workspace_id: self.workspace_id.clone(),
|
||||||
working_directory: self.working_directory.clone(),
|
working_directory: self.working_directory.clone(),
|
||||||
profile: self.request.profile.clone(),
|
profile: self.request.profile.clone(),
|
||||||
@@ -3043,6 +3020,7 @@ impl WorkerRecord {
|
|||||||
worker_ref: self.worker_ref.clone(),
|
worker_ref: self.worker_ref.clone(),
|
||||||
worker_id: self.worker_id,
|
worker_id: self.worker_id,
|
||||||
status: self.status,
|
status: self.status,
|
||||||
|
worker_state: self.worker_state.clone(),
|
||||||
workspace_id: self.workspace_id.clone(),
|
workspace_id: self.workspace_id.clone(),
|
||||||
working_directory: self.working_directory.clone(),
|
working_directory: self.working_directory.clone(),
|
||||||
profile: self.request.profile.clone(),
|
profile: self.request.profile.clone(),
|
||||||
@@ -3081,16 +3059,6 @@ fn restore_intent_for_status(status: WorkerStatus) -> WorkerRestoreIntent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_status_from_run_state(run_state: WorkerExecutionRunState) -> WorkerStatus {
|
|
||||||
match run_state {
|
|
||||||
WorkerExecutionRunState::Idle => WorkerStatus::Idle,
|
|
||||||
WorkerExecutionRunState::Busy => WorkerStatus::Running,
|
|
||||||
WorkerExecutionRunState::Stopped
|
|
||||||
| WorkerExecutionRunState::Rejected
|
|
||||||
| WorkerExecutionRunState::Errored => WorkerStatus::Stopped,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn repository_resource_error(error: BackendResourceError) -> RuntimeError {
|
fn repository_resource_error(error: BackendResourceError) -> RuntimeError {
|
||||||
let (code, message) = match error {
|
let (code, message) = match error {
|
||||||
BackendResourceError::Expired => (
|
BackendResourceError::Expired => (
|
||||||
@@ -3304,7 +3272,7 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use crate::execution::{
|
use crate::execution::{
|
||||||
WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle,
|
WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle,
|
||||||
WorkerExecutionRestoreRequest, WorkerExecutionRunState,
|
WorkerExecutionRestoreRequest,
|
||||||
};
|
};
|
||||||
use crate::working_directory::WorkingDirectoryDiagnostic;
|
use crate::working_directory::WorkingDirectoryDiagnostic;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
@@ -3313,6 +3281,14 @@ mod tests {
|
|||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
fn test_command() -> protocol::WorkerCommandEnvelope {
|
||||||
|
protocol::WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 1,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn repository_resource_failures_keep_typed_credential_diagnostics() {
|
fn repository_resource_failures_keep_typed_credential_diagnostics() {
|
||||||
let cases = [
|
let cases = [
|
||||||
@@ -3359,7 +3335,9 @@ mod tests {
|
|||||||
protocol::Event::InternalWorker {
|
protocol::Event::InternalWorker {
|
||||||
worker,
|
worker,
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: Box::new(protocol::Event::Status { status }),
|
event: Box::new(protocol::Event::WorkerState {
|
||||||
|
snapshot: status.into(),
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3452,7 +3430,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: protocol::WorkerStatus::Idle,
|
state: protocol::WorkerStatus::Idle.into(),
|
||||||
in_flight: protocol::InFlightSnapshot::default(),
|
in_flight: protocol::InFlightSnapshot::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
};
|
};
|
||||||
@@ -3967,7 +3945,6 @@ mod tests {
|
|||||||
.insert(request.worker_ref.worker_id.clone(), request.context);
|
.insert(request.worker_ref.worker_id.clone(), request.context);
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: request
|
working_directory: request
|
||||||
.working_directory
|
.working_directory
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -3997,7 +3974,6 @@ mod tests {
|
|||||||
.insert(request.worker_ref.worker_id.clone(), request.context);
|
.insert(request.worker_ref.worker_id.clone(), request.context);
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: request
|
working_directory: request
|
||||||
.working_directory
|
.working_directory
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -4020,7 +3996,6 @@ mod tests {
|
|||||||
.unwrap_or_else(|| {
|
.unwrap_or_else(|| {
|
||||||
WorkerExecutionResult::accepted_submission(
|
WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
"request-test",
|
"request-test",
|
||||||
"test-submission",
|
"test-submission",
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
@@ -4038,17 +4013,11 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
|
||||||
WorkerExecutionOperation::Stop,
|
|
||||||
WorkerExecutionRunState::Stopped,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Cancel)
|
||||||
WorkerExecutionOperation::Cancel,
|
|
||||||
WorkerExecutionRunState::Stopped,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
@@ -4374,7 +4343,9 @@ mod tests {
|
|||||||
.send_protocol_method_scoped(
|
.send_protocol_method_scoped(
|
||||||
&scope("workspace-a", "server-a"),
|
&scope("workspace-a", "server-a"),
|
||||||
&workspace_b.worker_ref,
|
&workspace_b.worker_ref,
|
||||||
Method::Shutdown,
|
Method::Shutdown {
|
||||||
|
command: test_command(),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
.unwrap_err();
|
.unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
@@ -4722,11 +4693,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn create_worker_uses_committed_input_ack_run_state() {
|
fn create_worker_does_not_infer_state_from_started_submission_ack() {
|
||||||
let (runtime, backend) = runtime_and_backend();
|
let (runtime, backend) = runtime_and_backend();
|
||||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
"request-test",
|
"request-test",
|
||||||
"test-submission",
|
"test-submission",
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
@@ -4737,6 +4707,68 @@ mod tests {
|
|||||||
let detail = runtime.create_worker(request).unwrap();
|
let detail = runtime.create_worker(request).unwrap();
|
||||||
|
|
||||||
assert_eq!(detail.status, WorkerStatus::Idle);
|
assert_eq!(detail.status, WorkerStatus::Idle);
|
||||||
|
assert_eq!(detail.worker_state, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_applies_only_newer_worker_state_snapshots() {
|
||||||
|
let (runtime, _) = runtime_and_backend();
|
||||||
|
let detail = runtime
|
||||||
|
.create_worker(task_request("state ordering"))
|
||||||
|
.unwrap();
|
||||||
|
let running = protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 7,
|
||||||
|
revision: 3,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 2,
|
||||||
|
};
|
||||||
|
assert!({
|
||||||
|
let mut state = runtime.lock().unwrap();
|
||||||
|
state.project_protocol_event_to_worker_state(
|
||||||
|
&detail.worker_ref,
|
||||||
|
&protocol::Event::WorkerState {
|
||||||
|
snapshot: running.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&detail.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.worker_state,
|
||||||
|
Some(running.clone())
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!({
|
||||||
|
let mut state = runtime.lock().unwrap();
|
||||||
|
!state.project_protocol_event_to_worker_state(
|
||||||
|
&detail.worker_ref,
|
||||||
|
&protocol::Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
revision: 2,
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..running.clone()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
});
|
||||||
|
assert!({
|
||||||
|
let mut state = runtime.lock().unwrap();
|
||||||
|
!state.project_protocol_event_to_worker_state(
|
||||||
|
&detail.worker_ref,
|
||||||
|
&protocol::Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..running.clone()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
});
|
||||||
|
let after = runtime.worker_detail(&detail.worker_ref).unwrap();
|
||||||
|
assert_eq!(after.status, WorkerStatus::Idle);
|
||||||
|
assert_eq!(after.worker_state, Some(running));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -4745,7 +4777,6 @@ mod tests {
|
|||||||
backend.preserve_commit_ack_submission_id();
|
backend.preserve_commit_ack_submission_id();
|
||||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Busy,
|
|
||||||
"request-test",
|
"request-test",
|
||||||
"forged-submission",
|
"forged-submission",
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
@@ -4770,7 +4801,6 @@ mod tests {
|
|||||||
let (runtime, backend) = runtime_and_backend();
|
let (runtime, backend) = runtime_and_backend();
|
||||||
backend.set_dispatch_result(WorkerExecutionResult::accepted(
|
backend.set_dispatch_result(WorkerExecutionResult::accepted(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Busy,
|
|
||||||
));
|
));
|
||||||
let mut request = task_request("missing initial input commit ack");
|
let mut request = task_request("missing initial input commit ack");
|
||||||
request.initial_input = Some(WorkerInput::user("start the ticket"));
|
request.initial_input = Some(WorkerInput::user("start the ticket"));
|
||||||
@@ -4898,7 +4928,7 @@ mod tests {
|
|||||||
context_window: 128,
|
context_window: 128,
|
||||||
context_tokens: 64,
|
context_tokens: 64,
|
||||||
},
|
},
|
||||||
status: protocol::WorkerStatus::Running,
|
state: protocol::WorkerStatus::Running.into(),
|
||||||
in_flight: protocol::InFlightSnapshot {
|
in_flight: protocol::InFlightSnapshot {
|
||||||
blocks: Vec::new(),
|
blocks: Vec::new(),
|
||||||
commands: Vec::new(),
|
commands: Vec::new(),
|
||||||
@@ -4914,18 +4944,38 @@ mod tests {
|
|||||||
protocol::Event::Snapshot {
|
protocol::Event::Snapshot {
|
||||||
session,
|
session,
|
||||||
greeting,
|
greeting,
|
||||||
status,
|
state,
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
assert_eq!(session.entries.len(), 1);
|
assert_eq!(session.entries.len(), 1);
|
||||||
assert_eq!(session.entries[0].entry_id, "restored-log-entry");
|
assert_eq!(session.entries[0].entry_id, "restored-log-entry");
|
||||||
assert_eq!(greeting.worker_name, "live-worker");
|
assert_eq!(greeting.worker_name, "live-worker");
|
||||||
assert_eq!(status, protocol::WorkerStatus::Running);
|
assert_eq!(state.catalog_status(), protocol::WorkerStatus::Running);
|
||||||
}
|
}
|
||||||
other => panic!("expected snapshot, got {other:?}"),
|
other => panic!("expected snapshot, got {other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "ws-server")]
|
||||||
|
#[test]
|
||||||
|
fn observation_snapshot_fails_closed_when_backend_snapshot_is_unavailable() {
|
||||||
|
let runtime = runtime_with_backend();
|
||||||
|
let detail = runtime
|
||||||
|
.create_worker(task_request("snapshot unavailable"))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
runtime
|
||||||
|
.worker_observation_snapshot(&detail.worker_ref)
|
||||||
|
.unwrap_err(),
|
||||||
|
RuntimeError::WorkerExecutionUnavailable {
|
||||||
|
worker_id,
|
||||||
|
message,
|
||||||
|
} if worker_id == detail.worker_ref.worker_id
|
||||||
|
&& message == "authoritative Worker snapshot is unavailable"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
struct InputOnlyBackend;
|
struct InputOnlyBackend;
|
||||||
|
|
||||||
impl WorkerExecutionBackend for InputOnlyBackend {
|
impl WorkerExecutionBackend for InputOnlyBackend {
|
||||||
@@ -4936,7 +4986,6 @@ mod tests {
|
|||||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: request
|
working_directory: request
|
||||||
.working_directory
|
.working_directory
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -4951,7 +5000,6 @@ mod tests {
|
|||||||
) -> WorkerExecutionResult {
|
) -> WorkerExecutionResult {
|
||||||
WorkerExecutionResult::accepted_submission(
|
WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
"request-test",
|
"request-test",
|
||||||
input
|
input
|
||||||
.submission_request_id
|
.submission_request_id
|
||||||
@@ -4993,7 +5041,12 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
runtime
|
runtime
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Shutdown)
|
.send_protocol_method(
|
||||||
|
&detail.worker_ref,
|
||||||
|
Method::Shutdown {
|
||||||
|
command: test_command(),
|
||||||
|
},
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -5009,7 +5062,12 @@ mod tests {
|
|||||||
.create_worker(task_request("restore explicitly"))
|
.create_worker(task_request("restore explicitly"))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
runtime
|
runtime
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Shutdown)
|
.send_protocol_method(
|
||||||
|
&detail.worker_ref,
|
||||||
|
Method::Shutdown {
|
||||||
|
command: test_command(),
|
||||||
|
},
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
@@ -5025,10 +5083,9 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
|
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
|
||||||
assert_eq!(
|
let restored = runtime.worker_detail(&detail.worker_ref).unwrap();
|
||||||
runtime.worker_detail(&detail.worker_ref).unwrap().status,
|
assert_eq!(restored.status, WorkerStatus::Idle);
|
||||||
WorkerStatus::Idle
|
assert_eq!(restored.worker_state, None);
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -10,8 +10,8 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, Mutex, mpsc};
|
use std::sync::{Arc, Mutex, RwLock, mpsc};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::auth::{
|
use crate::auth::{
|
||||||
@@ -25,8 +25,8 @@ use crate::catalog::{
|
|||||||
};
|
};
|
||||||
use crate::execution::{
|
use crate::execution::{
|
||||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation,
|
||||||
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState,
|
WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest,
|
||||||
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
WorkerExecutionSpawnResult,
|
||||||
};
|
};
|
||||||
use crate::identity::WorkerRef;
|
use crate::identity::WorkerRef;
|
||||||
use crate::interaction::{WorkerInput, WorkerInputKind};
|
use crate::interaction::{WorkerInput, WorkerInputKind};
|
||||||
@@ -38,7 +38,28 @@ use crate::working_directory::{
|
|||||||
WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer,
|
WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer,
|
||||||
};
|
};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus};
|
#[cfg(test)]
|
||||||
|
use protocol::WorkerStatus;
|
||||||
|
use protocol::{Event, Method, Segment, WorkerCommandEnvelope};
|
||||||
|
|
||||||
|
static NEXT_INTERNAL_COMMAND_ID: AtomicU64 = AtomicU64::new(1);
|
||||||
|
|
||||||
|
fn next_internal_command(
|
||||||
|
state: &RwLock<protocol::WorkerStateSnapshot>,
|
||||||
|
) -> Result<WorkerCommandEnvelope, String> {
|
||||||
|
let snapshot = state
|
||||||
|
.read()
|
||||||
|
.map_err(|_| "worker state lock is poisoned".to_string())?
|
||||||
|
.clone();
|
||||||
|
let floor = snapshot.last_command_id.saturating_add(1);
|
||||||
|
let command_id = NEXT_INTERNAL_COMMAND_ID
|
||||||
|
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
|
||||||
|
Some(current.max(floor).saturating_add(1))
|
||||||
|
})
|
||||||
|
.unwrap_or(floor)
|
||||||
|
.max(floor);
|
||||||
|
Ok(WorkerCommandEnvelope::for_snapshot(command_id, &snapshot))
|
||||||
|
}
|
||||||
use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
|
use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use session_store::{FsStore, FsWorkerStore};
|
use session_store::{FsStore, FsWorkerStore};
|
||||||
@@ -172,7 +193,7 @@ impl WorkerObservationProvider for RuntimeGrantedWorkerObservationProvider {
|
|||||||
},
|
},
|
||||||
display_name: grant.worker_id.clone(),
|
display_name: grant.worker_id.clone(),
|
||||||
relation: "granted_peer".to_string(),
|
relation: "granted_peer".to_string(),
|
||||||
status: format!("{:?}", state.get_status()).to_lowercase(),
|
status: format!("{:?}", state.catalog_status()).to_lowercase(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
subjects.sort_by(|left, right| left.subject.cmp(&right.subject));
|
subjects.sort_by(|left, right| left.subject.cmp(&right.subject));
|
||||||
@@ -1174,10 +1195,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
struct RuntimeWorkerExecution {
|
struct RuntimeWorkerExecution {
|
||||||
handle: WorkerHandle,
|
handle: WorkerHandle,
|
||||||
shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>,
|
shutdown: Arc<tokio::sync::Mutex<Option<worker::ShutdownReceiver>>>,
|
||||||
busy: Arc<AtomicBool>,
|
worker_state: Arc<RwLock<protocol::WorkerStateSnapshot>>,
|
||||||
workspace_client: Option<Arc<dyn WorkspaceClient>>,
|
workspace_client: Option<Arc<dyn WorkspaceClient>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1275,7 +1297,7 @@ where
|
|||||||
) -> Result<
|
) -> Result<
|
||||||
(
|
(
|
||||||
WorkerHandle,
|
WorkerHandle,
|
||||||
Arc<AtomicBool>,
|
Arc<RwLock<protocol::WorkerStateSnapshot>>,
|
||||||
Option<Arc<dyn WorkspaceClient>>,
|
Option<Arc<dyn WorkspaceClient>>,
|
||||||
),
|
),
|
||||||
WorkerExecutionResult,
|
WorkerExecutionResult,
|
||||||
@@ -1301,7 +1323,7 @@ where
|
|||||||
.map(|execution| {
|
.map(|execution| {
|
||||||
(
|
(
|
||||||
execution.handle.clone(),
|
execution.handle.clone(),
|
||||||
execution.busy.clone(),
|
execution.worker_state.clone(),
|
||||||
execution.workspace_client.clone(),
|
execution.workspace_client.clone(),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -1318,7 +1340,6 @@ where
|
|||||||
operation: WorkerExecutionOperation,
|
operation: WorkerExecutionOperation,
|
||||||
worker: WorkerHandle,
|
worker: WorkerHandle,
|
||||||
method: Method,
|
method: Method,
|
||||||
accepted_run_state: WorkerExecutionRunState,
|
|
||||||
) -> WorkerExecutionResult {
|
) -> WorkerExecutionResult {
|
||||||
self.run_on_adapter_runtime(async move {
|
self.run_on_adapter_runtime(async move {
|
||||||
worker
|
worker
|
||||||
@@ -1326,7 +1347,7 @@ where
|
|||||||
.await
|
.await
|
||||||
.map_err(|err| format!("failed to send Worker method: {err}"))
|
.map_err(|err| format!("failed to send Worker method: {err}"))
|
||||||
})
|
})
|
||||||
.map(|_| WorkerExecutionResult::accepted(operation, accepted_run_state))
|
.map(|_| WorkerExecutionResult::accepted(operation))
|
||||||
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
|
.unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1336,7 +1357,6 @@ where
|
|||||||
worker: WorkerHandle,
|
worker: WorkerHandle,
|
||||||
method: Method,
|
method: Method,
|
||||||
submission_request_id: String,
|
submission_request_id: String,
|
||||||
accepted_run_state: WorkerExecutionRunState,
|
|
||||||
) -> WorkerExecutionResult {
|
) -> WorkerExecutionResult {
|
||||||
let request_id = submission_request_id.clone();
|
let request_id = submission_request_id.clone();
|
||||||
self.run_on_adapter_runtime(async move {
|
self.run_on_adapter_runtime(async move {
|
||||||
@@ -1395,7 +1415,6 @@ where
|
|||||||
.map(|(submission_id, disposition)| {
|
.map(|(submission_id, disposition)| {
|
||||||
WorkerExecutionResult::accepted_submission(
|
WorkerExecutionResult::accepted_submission(
|
||||||
operation,
|
operation,
|
||||||
accepted_run_state,
|
|
||||||
submission_request_id,
|
submission_request_id,
|
||||||
submission_id,
|
submission_id,
|
||||||
disposition,
|
disposition,
|
||||||
@@ -1414,41 +1433,31 @@ where
|
|||||||
working_directory: Option<WorkingDirectoryBinding>,
|
working_directory: Option<WorkingDirectoryBinding>,
|
||||||
workspace_client: Option<Arc<dyn WorkspaceClient>>,
|
workspace_client: Option<Arc<dyn WorkspaceClient>>,
|
||||||
) -> WorkerExecutionSpawnResult {
|
) -> WorkerExecutionSpawnResult {
|
||||||
let busy = Arc::new(AtomicBool::new(false));
|
let worker_state = Arc::new(RwLock::new(handle.shared_state.snapshot()));
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
{
|
{
|
||||||
let streams = subscribe_worker_protocol_session(&handle);
|
let streams = subscribe_worker_protocol_session(&handle);
|
||||||
let mut events = streams.events;
|
let mut events = streams.events;
|
||||||
let mut entry_events = streams.log_entries;
|
let mut entry_events = streams.log_entries;
|
||||||
let bridge_busy = busy.clone();
|
let bridge_worker_state = worker_state.clone();
|
||||||
if let Err(message) = self.spawn_on_adapter_runtime(async move {
|
if let Err(message) = self.spawn_on_adapter_runtime(async move {
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
event = events.recv() => {
|
event = events.recv() => {
|
||||||
match event {
|
match event {
|
||||||
Ok(event) => {
|
Ok(mut event) => {
|
||||||
let next_busy = match &event {
|
match apply_protocol_worker_state(&bridge_worker_state, &mut event) {
|
||||||
Event::InvokeStart { .. }
|
Ok(true) => {
|
||||||
| Event::Status {
|
|
||||||
status: WorkerStatus::Running,
|
|
||||||
} => Some(true),
|
|
||||||
Event::RunEnd { .. }
|
|
||||||
| Event::Error {
|
|
||||||
code: ErrorCode::NotPaused,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
| Event::Status {
|
|
||||||
status:
|
|
||||||
WorkerStatus::Idle
|
|
||||||
| WorkerStatus::Paused
|
|
||||||
| WorkerStatus::Stopped,
|
|
||||||
}
|
|
||||||
| Event::Shutdown => Some(false),
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
let _ = bridge_context.publish_protocol_event(event);
|
let _ = bridge_context.publish_protocol_event(event);
|
||||||
if let Some(next_busy) = next_busy {
|
}
|
||||||
bridge_busy.store(next_busy, Ordering::SeqCst);
|
Ok(false) => {}
|
||||||
|
Err(message) => {
|
||||||
|
let _ = bridge_context.publish_protocol_event(Event::Error {
|
||||||
|
code: protocol::ErrorCode::Internal,
|
||||||
|
message: format!("worker state stream rejected: {message}"),
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||||
@@ -1493,14 +1502,13 @@ where
|
|||||||
RuntimeWorkerExecution {
|
RuntimeWorkerExecution {
|
||||||
handle,
|
handle,
|
||||||
shutdown,
|
shutdown,
|
||||||
busy,
|
worker_state,
|
||||||
workspace_client,
|
workspace_client,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
WorkerExecutionSpawnResult::Connected {
|
WorkerExecutionSpawnResult::Connected {
|
||||||
handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()),
|
handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: working_directory.map(|binding| binding.status()),
|
working_directory: working_directory.map(|binding| binding.status()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1516,45 +1524,28 @@ impl<F> Drop for WorkerRuntimeExecutionBackend<F> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn method_starts_turn(method: &Method) -> bool {
|
fn apply_protocol_worker_state(
|
||||||
matches!(
|
current: &Arc<RwLock<protocol::WorkerStateSnapshot>>,
|
||||||
method,
|
event: &mut Event,
|
||||||
Method::Submit { .. }
|
) -> Result<bool, String> {
|
||||||
| Method::SubmitTracked { .. }
|
let (incoming, replace_stale) = match event {
|
||||||
| Method::Notify { auto_run: true, .. }
|
Event::WorkerState { snapshot } => (snapshot, false),
|
||||||
| Method::NotifyTracked { auto_run: true, .. }
|
Event::Snapshot { state, .. } => (state, true),
|
||||||
| Method::Resume
|
Event::CommandAcknowledged { acknowledgement } => (&mut acknowledgement.state, true),
|
||||||
| Method::Compact
|
_ => return Ok(true),
|
||||||
)
|
};
|
||||||
|
let mut current = current
|
||||||
|
.write()
|
||||||
|
.map_err(|_| "worker state projection lock is poisoned".to_string())?;
|
||||||
|
match protocol::apply_worker_state_snapshot(&mut current, incoming) {
|
||||||
|
Ok(protocol::WorkerStateSnapshotApply::Applied)
|
||||||
|
| Ok(protocol::WorkerStateSnapshotApply::Duplicate) => Ok(true),
|
||||||
|
Ok(protocol::WorkerStateSnapshotApply::Stale) if replace_stale => {
|
||||||
|
*incoming = current.clone();
|
||||||
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
Ok(protocol::WorkerStateSnapshotApply::Stale) => Ok(false),
|
||||||
fn method_can_start_turn_from_status(method: &Method, status: WorkerStatus) -> bool {
|
Err(error) => Err(error.to_string()),
|
||||||
match method {
|
|
||||||
Method::Resume => matches!(status, WorkerStatus::Idle | WorkerStatus::Paused),
|
|
||||||
_ => status == WorkerStatus::Idle,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExecutionRunState {
|
|
||||||
match status {
|
|
||||||
WorkerStatus::Running => WorkerExecutionRunState::Busy,
|
|
||||||
WorkerStatus::Idle if auto_run => WorkerExecutionRunState::Busy,
|
|
||||||
WorkerStatus::Idle | WorkerStatus::Paused | WorkerStatus::Stopped => {
|
|
||||||
WorkerExecutionRunState::Idle
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState {
|
|
||||||
match method {
|
|
||||||
Method::Submit { .. }
|
|
||||||
| Method::SubmitTracked { .. }
|
|
||||||
| Method::Notify { auto_run: true, .. }
|
|
||||||
| Method::NotifyTracked { auto_run: true, .. }
|
|
||||||
| Method::Resume
|
|
||||||
| Method::Compact => WorkerExecutionRunState::Busy,
|
|
||||||
Method::Shutdown => WorkerExecutionRunState::Stopped,
|
|
||||||
_ => WorkerExecutionRunState::Idle,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1883,7 +1874,7 @@ where
|
|||||||
handle: &WorkerExecutionHandle,
|
handle: &WorkerExecutionHandle,
|
||||||
input: WorkerInput,
|
input: WorkerInput,
|
||||||
) -> WorkerExecutionResult {
|
) -> WorkerExecutionResult {
|
||||||
let (worker, busy, _workspace_client) = match self.get_execution(handle) {
|
let (worker, worker_state, _workspace_client) = match self.get_execution(handle) {
|
||||||
Ok(execution) => execution,
|
Ok(execution) => execution,
|
||||||
Err(mut result) => {
|
Err(mut result) => {
|
||||||
result.operation = WorkerExecutionOperation::Input;
|
result.operation = WorkerExecutionOperation::Input;
|
||||||
@@ -1892,16 +1883,10 @@ where
|
|||||||
};
|
};
|
||||||
|
|
||||||
if input.kind == WorkerInputKind::Notify {
|
if input.kind == WorkerInputKind::Notify {
|
||||||
let status = worker.shared_state.get_status();
|
|
||||||
let accepted_run_state = accepted_notify_run_state(status, true);
|
|
||||||
let claimed_here = status == WorkerStatus::Idle
|
|
||||||
&& busy
|
|
||||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
|
||||||
.is_ok();
|
|
||||||
let notification_request_id = input
|
let notification_request_id = input
|
||||||
.submission_request_id
|
.submission_request_id
|
||||||
.unwrap_or_else(protocol::new_submission_request_id);
|
.unwrap_or_else(protocol::new_submission_request_id);
|
||||||
let result = self.send_method(
|
return self.send_method(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
worker,
|
worker,
|
||||||
Method::NotifyTracked {
|
Method::NotifyTracked {
|
||||||
@@ -1912,25 +1897,20 @@ where
|
|||||||
operation_id: notification_request_id,
|
operation_id: notification_request_id,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
accepted_run_state,
|
|
||||||
);
|
);
|
||||||
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
|
||||||
{
|
|
||||||
busy.store(false, Ordering::SeqCst);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let is_user_submit = input.kind == WorkerInputKind::User;
|
if input.kind == WorkerInputKind::Compact {
|
||||||
let status = worker.shared_state.get_status();
|
let command = match next_internal_command(&worker_state) {
|
||||||
let claimed_here = status == WorkerStatus::Idle
|
Ok(command) => command,
|
||||||
&& busy
|
Err(error) => {
|
||||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
return WorkerExecutionResult::errored(WorkerExecutionOperation::Input, error);
|
||||||
.is_ok();
|
}
|
||||||
if !is_user_submit && !claimed_here {
|
};
|
||||||
return WorkerExecutionResult::busy(
|
return self.send_method(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
"Worker is already running",
|
worker,
|
||||||
|
Method::Compact { command },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1940,7 +1920,6 @@ where
|
|||||||
.submission_request_id
|
.submission_request_id
|
||||||
.filter(|submission_id| !submission_id.trim().is_empty())
|
.filter(|submission_id| !submission_id.trim().is_empty())
|
||||||
else {
|
else {
|
||||||
busy.store(false, Ordering::SeqCst);
|
|
||||||
return WorkerExecutionResult::rejected(
|
return WorkerExecutionResult::rejected(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
"Runtime user input is missing its internal submission id",
|
"Runtime user input is missing its internal submission id",
|
||||||
@@ -1960,9 +1939,9 @@ where
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
WorkerInputKind::Notify => {
|
WorkerInputKind::Notify => {
|
||||||
unreachable!("Notify input is dispatched before the turn-start busy guard")
|
unreachable!("Notify input is dispatched before ordinary input mapping")
|
||||||
}
|
}
|
||||||
WorkerInputKind::Compact => (Method::Compact, None),
|
WorkerInputKind::Compact => unreachable!("compact input is dispatched above"),
|
||||||
WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None),
|
WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None),
|
||||||
WorkerInputKind::RegisterPeer => (
|
WorkerInputKind::RegisterPeer => (
|
||||||
Method::RegisterPeer {
|
Method::RegisterPeer {
|
||||||
@@ -1971,40 +1950,18 @@ where
|
|||||||
None,
|
None,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
let accepted_run_state = match method {
|
|
||||||
Method::Submit { .. }
|
|
||||||
| Method::SubmitTracked { .. }
|
|
||||||
| Method::Notify { .. }
|
|
||||||
| Method::NotifyTracked { .. }
|
|
||||||
| Method::Compact => WorkerExecutionRunState::Busy,
|
|
||||||
_ => WorkerExecutionRunState::Idle,
|
|
||||||
};
|
|
||||||
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
|
|
||||||
let waits_for_submission_acceptance = submission_request_id.is_some();
|
let waits_for_submission_acceptance = submission_request_id.is_some();
|
||||||
|
|
||||||
let result = if waits_for_submission_acceptance {
|
if waits_for_submission_acceptance {
|
||||||
self.send_submit_and_wait_for_acceptance(
|
self.send_submit_and_wait_for_acceptance(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
worker,
|
worker,
|
||||||
method,
|
method,
|
||||||
submission_request_id.expect("Submit must have a submission request id"),
|
submission_request_id.expect("Submit must have a submission request id"),
|
||||||
accepted_run_state,
|
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
self.send_method(
|
self.send_method(WorkerExecutionOperation::Input, worker, method)
|
||||||
WorkerExecutionOperation::Input,
|
|
||||||
worker,
|
|
||||||
method,
|
|
||||||
accepted_run_state,
|
|
||||||
)
|
|
||||||
};
|
|
||||||
if accepted_is_idle
|
|
||||||
|| (claimed_here
|
|
||||||
&& result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
|
|
||||||
{
|
|
||||||
busy.store(false, Ordering::SeqCst);
|
|
||||||
}
|
}
|
||||||
result
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn upload_file(
|
fn upload_file(
|
||||||
@@ -2046,10 +2003,7 @@ where
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
match worker.delete_uploaded_file(artifact_id) {
|
match worker.delete_uploaded_file(artifact_id) {
|
||||||
Ok(_) => WorkerExecutionResult::accepted(
|
Ok(_) => WorkerExecutionResult::accepted(WorkerExecutionOperation::DeleteUploadedFile),
|
||||||
WorkerExecutionOperation::DeleteUploadedFile,
|
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
),
|
|
||||||
Err(error) => WorkerExecutionResult::rejected(
|
Err(error) => WorkerExecutionResult::rejected(
|
||||||
WorkerExecutionOperation::DeleteUploadedFile,
|
WorkerExecutionOperation::DeleteUploadedFile,
|
||||||
format!("uploaded_file_delete_rejected: {error}"),
|
format!("uploaded_file_delete_rejected: {error}"),
|
||||||
@@ -2062,7 +2016,7 @@ where
|
|||||||
handle: &WorkerExecutionHandle,
|
handle: &WorkerExecutionHandle,
|
||||||
method: Method,
|
method: Method,
|
||||||
) -> WorkerExecutionResult {
|
) -> WorkerExecutionResult {
|
||||||
let (worker, busy, _workspace_client) = match self.get_execution(handle) {
|
let (worker, _worker_state, _workspace_client) = match self.get_execution(handle) {
|
||||||
Ok(execution) => execution,
|
Ok(execution) => execution,
|
||||||
Err(mut result) => {
|
Err(mut result) => {
|
||||||
result.operation = WorkerExecutionOperation::ProtocolMethod;
|
result.operation = WorkerExecutionOperation::ProtocolMethod;
|
||||||
@@ -2070,59 +2024,7 @@ where
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(auto_run) = match &method {
|
self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method)
|
||||||
Method::Notify { auto_run, .. } | Method::NotifyTracked { auto_run, .. } => {
|
|
||||||
Some(*auto_run)
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
} {
|
|
||||||
let status = worker.shared_state.get_status();
|
|
||||||
let accepted_run_state = accepted_notify_run_state(status, auto_run);
|
|
||||||
let claimed_here = status == WorkerStatus::Idle
|
|
||||||
&& auto_run
|
|
||||||
&& busy
|
|
||||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
|
||||||
.is_ok();
|
|
||||||
let result = self.send_method(
|
|
||||||
WorkerExecutionOperation::ProtocolMethod,
|
|
||||||
worker,
|
|
||||||
method,
|
|
||||||
accepted_run_state,
|
|
||||||
);
|
|
||||||
if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted
|
|
||||||
{
|
|
||||||
busy.store(false, Ordering::SeqCst);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
let starts_turn = method_starts_turn(&method);
|
|
||||||
if starts_turn
|
|
||||||
&& (!method_can_start_turn_from_status(&method, worker.shared_state.get_status())
|
|
||||||
|| busy
|
|
||||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
|
||||||
.is_err())
|
|
||||||
{
|
|
||||||
return WorkerExecutionResult::busy(
|
|
||||||
WorkerExecutionOperation::ProtocolMethod,
|
|
||||||
"Worker is already running; runtime adapter v0 does not queue protocol methods",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let accepted_run_state = accepted_run_state_for_method(&method);
|
|
||||||
let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle;
|
|
||||||
let result = self.send_method(
|
|
||||||
WorkerExecutionOperation::ProtocolMethod,
|
|
||||||
worker,
|
|
||||||
method,
|
|
||||||
accepted_run_state,
|
|
||||||
);
|
|
||||||
if (starts_turn && accepted_is_idle)
|
|
||||||
|| (starts_turn && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted)
|
|
||||||
{
|
|
||||||
busy.store(false, Ordering::SeqCst);
|
|
||||||
}
|
|
||||||
result
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||||
@@ -2137,7 +2039,7 @@ where
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
let execution = match self.workers.lock() {
|
let execution = match self.workers.lock() {
|
||||||
Ok(mut workers) => workers.remove(handle.worker_ref()),
|
Ok(workers) => workers.get(handle.worker_ref()).cloned(),
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
return WorkerExecutionResult::errored(
|
return WorkerExecutionResult::errored(
|
||||||
WorkerExecutionOperation::Stop,
|
WorkerExecutionOperation::Stop,
|
||||||
@@ -2153,48 +2055,73 @@ where
|
|||||||
};
|
};
|
||||||
let artifact_cleanup = execution.handle.clone();
|
let artifact_cleanup = execution.handle.clone();
|
||||||
let shutdown = execution.shutdown.clone();
|
let shutdown = execution.shutdown.clone();
|
||||||
|
let command = match next_internal_command(&execution.worker_state) {
|
||||||
|
Ok(command) => command,
|
||||||
|
Err(error) => {
|
||||||
|
return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, error);
|
||||||
|
}
|
||||||
|
};
|
||||||
let result = self.send_method(
|
let result = self.send_method(
|
||||||
WorkerExecutionOperation::Stop,
|
WorkerExecutionOperation::Stop,
|
||||||
execution.handle,
|
execution.handle.clone(),
|
||||||
Method::Shutdown,
|
Method::Shutdown { command },
|
||||||
WorkerExecutionRunState::Stopped,
|
|
||||||
);
|
);
|
||||||
if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted {
|
if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
match self.run_on_adapter_runtime(async move {
|
let shutdown_wait = self.run_on_adapter_runtime(async move {
|
||||||
let receiver = shutdown.lock().await.take();
|
let mut guard = shutdown.lock().await;
|
||||||
if let Some(receiver) = receiver {
|
let Some(mut receiver) = guard.take() else {
|
||||||
receiver
|
return Ok(());
|
||||||
.await
|
};
|
||||||
.map_err(|_| "Worker shutdown completion channel closed".to_string())?;
|
match tokio::time::timeout(Duration::from_secs(5), &mut receiver).await {
|
||||||
|
Ok(Ok(())) => Ok(()),
|
||||||
|
Ok(Err(_)) => Err("Worker shutdown completion channel closed".to_string()),
|
||||||
|
Err(_) => {
|
||||||
|
*guard = Some(receiver);
|
||||||
|
Err("Worker shutdown confirmation timed out; stop remains retryable".into())
|
||||||
}
|
}
|
||||||
Ok(())
|
}
|
||||||
}) {
|
});
|
||||||
Ok(()) => match artifact_cleanup.delete_uncommitted_uploaded_files() {
|
if let Err(message) = shutdown_wait {
|
||||||
Ok(_) => result,
|
return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message);
|
||||||
Err(error) => WorkerExecutionResult::errored(
|
}
|
||||||
|
if let Err(error) = artifact_cleanup.delete_uncommitted_uploaded_files() {
|
||||||
|
return WorkerExecutionResult::errored(
|
||||||
WorkerExecutionOperation::Stop,
|
WorkerExecutionOperation::Stop,
|
||||||
format!("uploaded_file_cleanup_failed: {error}"),
|
format!("uploaded_file_cleanup_failed: {error}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
match self.workers.lock() {
|
||||||
|
Ok(mut workers) => {
|
||||||
|
workers.remove(handle.worker_ref());
|
||||||
|
result
|
||||||
|
}
|
||||||
|
Err(_) => WorkerExecutionResult::errored(
|
||||||
|
WorkerExecutionOperation::Stop,
|
||||||
|
"worker adapter registry lock is poisoned after shutdown",
|
||||||
),
|
),
|
||||||
},
|
|
||||||
Err(message) => WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||||
let (worker, _busy, _workspace_client) = match self.get_execution(handle) {
|
let (worker, worker_state, _workspace_client) = match self.get_execution(handle) {
|
||||||
Ok(execution) => execution,
|
Ok(execution) => execution,
|
||||||
Err(mut result) => {
|
Err(mut result) => {
|
||||||
result.operation = WorkerExecutionOperation::Cancel;
|
result.operation = WorkerExecutionOperation::Cancel;
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let command = match next_internal_command(&worker_state) {
|
||||||
|
Ok(command) => command,
|
||||||
|
Err(error) => {
|
||||||
|
return WorkerExecutionResult::errored(WorkerExecutionOperation::Cancel, error);
|
||||||
|
}
|
||||||
|
};
|
||||||
self.send_method(
|
self.send_method(
|
||||||
WorkerExecutionOperation::Cancel,
|
WorkerExecutionOperation::Cancel,
|
||||||
worker,
|
worker,
|
||||||
Method::Cancel,
|
Method::Cancel { command },
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2259,6 +2186,79 @@ mod tests {
|
|||||||
use manifest::{Scope, WorkerManifest};
|
use manifest::{Scope, WorkerManifest};
|
||||||
use session_store::{LogEntry, WorkerMetadataStore};
|
use session_store::{LogEntry, WorkerMetadataStore};
|
||||||
|
|
||||||
|
fn test_command() -> WorkerCommandEnvelope {
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 1,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn adapter_command(
|
||||||
|
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> WorkerCommandEnvelope {
|
||||||
|
let workers = backend.workers.lock().unwrap();
|
||||||
|
let state = workers
|
||||||
|
.get(worker_ref)
|
||||||
|
.expect("worker execution")
|
||||||
|
.worker_state
|
||||||
|
.read()
|
||||||
|
.unwrap()
|
||||||
|
.clone();
|
||||||
|
WorkerCommandEnvelope::for_snapshot(state.last_command_id.saturating_add(1), &state)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn protocol_bridge_applies_state_and_acknowledgement_monotonically() {
|
||||||
|
let running = protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 4,
|
||||||
|
revision: 3,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 2,
|
||||||
|
};
|
||||||
|
let current = Arc::new(RwLock::new(running.clone()));
|
||||||
|
let mut stale = Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
revision: 2,
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..running.clone()
|
||||||
|
},
|
||||||
|
};
|
||||||
|
assert!(!apply_protocol_worker_state(¤t, &mut stale).unwrap());
|
||||||
|
assert_eq!(*current.read().unwrap(), running);
|
||||||
|
|
||||||
|
let paused = protocol::WorkerStateSnapshot {
|
||||||
|
revision: 4,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Paused,
|
||||||
|
)),
|
||||||
|
last_command_id: 3,
|
||||||
|
..running.clone()
|
||||||
|
};
|
||||||
|
let mut acknowledgement = Event::CommandAcknowledged {
|
||||||
|
acknowledgement: protocol::WorkerCommandAcknowledgement {
|
||||||
|
command_id: 3,
|
||||||
|
command: protocol::WorkerCommandKind::Pause,
|
||||||
|
disposition: protocol::WorkerCommandDisposition::Accepted,
|
||||||
|
state: paused.clone(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
assert!(apply_protocol_worker_state(¤t, &mut acknowledgement).unwrap());
|
||||||
|
assert_eq!(*current.read().unwrap(), paused);
|
||||||
|
|
||||||
|
let mut conflict = Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..paused.clone()
|
||||||
|
},
|
||||||
|
};
|
||||||
|
assert!(apply_protocol_worker_state(¤t, &mut conflict).is_err());
|
||||||
|
assert_eq!(*current.read().unwrap(), paused);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn workspace_prompt_projection_notification_advances_shared_cache() {
|
fn workspace_prompt_projection_notification_advances_shared_cache() {
|
||||||
let cache = WorkspacePromptProjectionCache::default();
|
let cache = WorkspacePromptProjectionCache::default();
|
||||||
@@ -2405,46 +2405,6 @@ mod tests {
|
|||||||
assert_eq!(after_restore_workspace_id.as_deref(), Some("workspace-a"));
|
assert_eq!(after_restore_workspace_id.as_deref(), Some("workspace-a"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn notify_run_state_allows_running_worker_inbox_delivery() {
|
|
||||||
assert_eq!(
|
|
||||||
accepted_notify_run_state(WorkerStatus::Running, true),
|
|
||||||
WorkerExecutionRunState::Busy
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
accepted_notify_run_state(WorkerStatus::Idle, true),
|
|
||||||
WorkerExecutionRunState::Busy
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
accepted_notify_run_state(WorkerStatus::Idle, false),
|
|
||||||
WorkerExecutionRunState::Idle
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
accepted_notify_run_state(WorkerStatus::Paused, true),
|
|
||||||
WorkerExecutionRunState::Idle
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn resume_turn_claim_accepts_paused_and_idle_but_not_running_status() {
|
|
||||||
assert!(method_can_start_turn_from_status(
|
|
||||||
&Method::Resume,
|
|
||||||
WorkerStatus::Paused
|
|
||||||
));
|
|
||||||
assert!(method_can_start_turn_from_status(
|
|
||||||
&Method::Resume,
|
|
||||||
WorkerStatus::Idle
|
|
||||||
));
|
|
||||||
assert!(!method_can_start_turn_from_status(
|
|
||||||
&Method::Resume,
|
|
||||||
WorkerStatus::Running
|
|
||||||
));
|
|
||||||
assert!(!method_can_start_turn_from_status(
|
|
||||||
&Method::Compact,
|
|
||||||
WorkerStatus::Paused
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
enum MockResponse {
|
enum MockResponse {
|
||||||
Complete(Vec<LlmEvent>),
|
Complete(Vec<LlmEvent>),
|
||||||
@@ -2645,28 +2605,53 @@ mod tests {
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn wait_for_adapter_command(
|
||||||
|
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
expected_command_id: u64,
|
||||||
|
) {
|
||||||
|
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||||
|
loop {
|
||||||
|
let observed = {
|
||||||
|
let workers = backend.workers.lock().unwrap();
|
||||||
|
workers
|
||||||
|
.get(worker_ref)
|
||||||
|
.expect("live Worker execution")
|
||||||
|
.worker_state
|
||||||
|
.read()
|
||||||
|
.unwrap()
|
||||||
|
.last_command_id
|
||||||
|
};
|
||||||
|
if observed >= expected_command_id {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
std::time::Instant::now() < deadline,
|
||||||
|
"timed out waiting for adapter command {expected_command_id}; last observed={observed}",
|
||||||
|
);
|
||||||
|
std::thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn wait_for_adapter_state(
|
fn wait_for_adapter_state(
|
||||||
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
|
backend: &WorkerRuntimeExecutionBackend<MockFactory>,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
expected_status: WorkerStatus,
|
expected_status: WorkerStatus,
|
||||||
expected_busy: bool,
|
|
||||||
) {
|
) {
|
||||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||||
loop {
|
loop {
|
||||||
let observed = {
|
let observed = {
|
||||||
let workers = backend.workers.lock().unwrap();
|
let workers = backend.workers.lock().unwrap();
|
||||||
let execution = workers.get(worker_ref).expect("live Worker execution");
|
let execution = workers.get(worker_ref).expect("live Worker execution");
|
||||||
(
|
let projected = execution.worker_state.read().unwrap().catalog_status();
|
||||||
execution.handle.shared_state.get_status(),
|
(execution.handle.shared_state.catalog_status(), projected)
|
||||||
execution.busy.load(Ordering::SeqCst),
|
|
||||||
)
|
|
||||||
};
|
};
|
||||||
if observed == (expected_status, expected_busy) {
|
if observed == (expected_status, expected_status) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
std::time::Instant::now() < deadline,
|
std::time::Instant::now() < deadline,
|
||||||
"timed out waiting for adapter state {expected_status:?}, busy={expected_busy}; last observed status={:?}, busy={}",
|
"timed out waiting for adapter state {expected_status:?}; last observed controller={:?}, projected={:?}",
|
||||||
observed.0,
|
observed.0,
|
||||||
observed.1,
|
observed.1,
|
||||||
);
|
);
|
||||||
@@ -3169,13 +3154,19 @@ mod tests {
|
|||||||
.expect("in-process restore must not bind the overlong Unix socket path");
|
.expect("in-process restore must not bind the overlong Unix socket path");
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
controller.handle.shared_state.get_status(),
|
controller.handle.shared_state.catalog_status(),
|
||||||
WorkerStatus::Idle
|
WorkerStatus::Idle
|
||||||
);
|
);
|
||||||
assert!(!socket_path.exists());
|
assert!(!socket_path.exists());
|
||||||
assert!(run_dir.join("worker.out.log").is_file());
|
assert!(run_dir.join("worker.out.log").is_file());
|
||||||
assert!(run_dir.join("worker.err.log").is_file());
|
assert!(run_dir.join("worker.err.log").is_file());
|
||||||
controller.handle.send(Method::Shutdown).await.unwrap();
|
controller
|
||||||
|
.handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: test_command(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
if let Some(receiver) = controller.shutdown.lock().await.take() {
|
if let Some(receiver) = controller.shutdown.lock().await.take() {
|
||||||
receiver.await.unwrap();
|
receiver.await.unwrap();
|
||||||
}
|
}
|
||||||
@@ -3289,7 +3280,9 @@ mod tests {
|
|||||||
backend
|
backend
|
||||||
.run_on_adapter_runtime(async move {
|
.run_on_adapter_runtime(async move {
|
||||||
handle
|
handle
|
||||||
.send(Method::Shutdown)
|
.send(Method::Shutdown {
|
||||||
|
command: test_command(),
|
||||||
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|error| error.to_string())?;
|
.map_err(|error| error.to_string())?;
|
||||||
if let Some(receiver) = shutdown.lock().await.take() {
|
if let Some(receiver) = shutdown.lock().await.take() {
|
||||||
@@ -3614,6 +3607,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
fn adapter_resumes_paused_turn_once_and_preserves_idle_not_paused_error() {
|
fn adapter_resumes_paused_turn_once_and_preserves_idle_not_paused_error() {
|
||||||
let hanging_events = || simple_text_events().into_iter().take(2).collect::<Vec<_>>();
|
let hanging_events = || simple_text_events().into_iter().take(2).collect::<Vec<_>>();
|
||||||
let client = MockClient::sequential(vec![
|
let client = MockClient::sequential(vec![
|
||||||
@@ -3646,62 +3640,91 @@ mod tests {
|
|||||||
runtime
|
runtime
|
||||||
.send_input(&detail.worker_ref, WorkerInput::user("pause and resume"))
|
.send_input(&detail.worker_ref, WorkerInput::user("pause and resume"))
|
||||||
.expect("start initial turn");
|
.expect("start initial turn");
|
||||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true);
|
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running);
|
||||||
|
|
||||||
let running_resume = runtime
|
let running_resume = adapter_command(&backend, &detail.worker_ref);
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
runtime
|
||||||
.expect_err("Resume while Running must be rejected");
|
.send_protocol_method(
|
||||||
assert!(
|
&detail.worker_ref,
|
||||||
running_resume
|
Method::Resume {
|
||||||
.to_string()
|
command: running_resume,
|
||||||
.contains("does not queue protocol methods"),
|
},
|
||||||
"unexpected Running Resume error: {running_resume}"
|
)
|
||||||
);
|
.expect("running Resume is forwarded for controller admission");
|
||||||
|
wait_for_adapter_command(&backend, &detail.worker_ref, running_resume.command_id);
|
||||||
|
|
||||||
runtime
|
runtime
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Pause)
|
.send_protocol_method(
|
||||||
|
&detail.worker_ref,
|
||||||
|
Method::Pause {
|
||||||
|
command: adapter_command(&backend, &detail.worker_ref),
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("pause initial turn");
|
.expect("pause initial turn");
|
||||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false);
|
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused);
|
||||||
|
|
||||||
runtime
|
runtime
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
.send_protocol_method(
|
||||||
|
&detail.worker_ref,
|
||||||
|
Method::Resume {
|
||||||
|
command: adapter_command(&backend, &detail.worker_ref),
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("resume paused turn");
|
.expect("resume paused turn");
|
||||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true);
|
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running);
|
||||||
|
|
||||||
let duplicate_resume = runtime
|
let duplicate_resume = adapter_command(&backend, &detail.worker_ref);
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
runtime
|
||||||
.expect_err("duplicate Resume must be rejected");
|
.send_protocol_method(
|
||||||
assert!(
|
&detail.worker_ref,
|
||||||
duplicate_resume
|
Method::Resume {
|
||||||
.to_string()
|
command: duplicate_resume,
|
||||||
.contains("does not queue protocol methods"),
|
},
|
||||||
"unexpected duplicate Resume error: {duplicate_resume}"
|
)
|
||||||
);
|
.expect("duplicate Resume is forwarded for controller admission");
|
||||||
|
wait_for_adapter_command(&backend, &detail.worker_ref, duplicate_resume.command_id);
|
||||||
|
|
||||||
runtime
|
runtime
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Pause)
|
.send_protocol_method(
|
||||||
|
&detail.worker_ref,
|
||||||
|
Method::Pause {
|
||||||
|
command: adapter_command(&backend, &detail.worker_ref),
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("pause resumed turn");
|
.expect("pause resumed turn");
|
||||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false);
|
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused);
|
||||||
runtime
|
runtime
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
.send_protocol_method(
|
||||||
|
&detail.worker_ref,
|
||||||
|
Method::Resume {
|
||||||
|
command: adapter_command(&backend, &detail.worker_ref),
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("resume paused turn a second time");
|
.expect("resume paused turn a second time");
|
||||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false);
|
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle);
|
||||||
assert_eq!(call_count.load(Ordering::SeqCst), 3);
|
assert_eq!(call_count.load(Ordering::SeqCst), 3);
|
||||||
|
|
||||||
|
let idle_resume = adapter_command(&backend, &detail.worker_ref);
|
||||||
runtime
|
runtime
|
||||||
.send_protocol_method(&detail.worker_ref, Method::Resume)
|
.send_protocol_method(
|
||||||
|
&detail.worker_ref,
|
||||||
|
Method::Resume {
|
||||||
|
command: idle_resume,
|
||||||
|
},
|
||||||
|
)
|
||||||
.expect("Idle Resume preserves controller NotPaused semantics");
|
.expect("Idle Resume preserves controller NotPaused semantics");
|
||||||
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false);
|
wait_for_adapter_command(&backend, &detail.worker_ref, idle_resume.command_id);
|
||||||
|
wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle);
|
||||||
let events = runtime
|
let events = runtime
|
||||||
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
|
.read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero())
|
||||||
.expect("read protocol events");
|
.expect("read protocol events");
|
||||||
assert!(events.iter().any(|event| {
|
assert!(events.iter().any(|event| {
|
||||||
matches!(
|
matches!(
|
||||||
&event.payload,
|
&event.payload,
|
||||||
Event::Error {
|
Event::CommandAcknowledged { acknowledgement }
|
||||||
code: protocol::ErrorCode::NotPaused,
|
if acknowledgement.command == protocol::WorkerCommandKind::Resume
|
||||||
..
|
&& acknowledgement.disposition
|
||||||
}
|
== protocol::WorkerCommandDisposition::InvalidState
|
||||||
)
|
)
|
||||||
}));
|
}));
|
||||||
assert_eq!(call_count.load(Ordering::SeqCst), 3);
|
assert_eq!(call_count.load(Ordering::SeqCst), 3);
|
||||||
|
|||||||
+649
-96
@@ -1,3 +1,4 @@
|
|||||||
|
use std::collections::VecDeque;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
@@ -16,7 +17,7 @@ use crate::ipc::notify_buffer::NotifyBuffer;
|
|||||||
use crate::ipc::server::SocketServer;
|
use crate::ipc::server::SocketServer;
|
||||||
use crate::runtime::dir::RuntimeDir;
|
use crate::runtime::dir::RuntimeDir;
|
||||||
use crate::segment_log_sink::SegmentLogSink;
|
use crate::segment_log_sink::SegmentLogSink;
|
||||||
use crate::shared_state::WorkerSharedState;
|
use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState};
|
||||||
use crate::shutdown_after_idle::{
|
use crate::shutdown_after_idle::{
|
||||||
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
||||||
take_shutdown_request_after_status,
|
take_shutdown_request_after_status,
|
||||||
@@ -28,7 +29,9 @@ use protocol::{
|
|||||||
AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent,
|
AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent,
|
||||||
CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus,
|
CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus,
|
||||||
CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice,
|
CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice,
|
||||||
ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, WorkerStatus,
|
ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef,
|
||||||
|
WorkerBusyState, WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope,
|
||||||
|
WorkerCommandKind, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStatus,
|
||||||
};
|
};
|
||||||
use workdir::{
|
use workdir::{
|
||||||
CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot,
|
CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot,
|
||||||
@@ -138,7 +141,7 @@ impl WorkerHandle {
|
|||||||
let event = Event::Snapshot {
|
let event = Event::Snapshot {
|
||||||
session,
|
session,
|
||||||
greeting: self.shared_state.greeting.clone(),
|
greeting: self.shared_state.greeting.clone(),
|
||||||
status: self.shared_state.get_status(),
|
state: self.shared_state.snapshot(),
|
||||||
in_flight,
|
in_flight,
|
||||||
internal_workers: self.spawned_registry.internal_worker_snapshots(),
|
internal_workers: self.spawned_registry.internal_worker_snapshots(),
|
||||||
};
|
};
|
||||||
@@ -178,15 +181,101 @@ impl WorkerHandle {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn command_admission_disposition(
|
||||||
|
admission: WorkerCommandAdmission,
|
||||||
|
) -> Result<(), WorkerCommandDisposition> {
|
||||||
|
match admission {
|
||||||
|
WorkerCommandAdmission::Accepted => Ok(()),
|
||||||
|
WorkerCommandAdmission::Retry | WorkerCommandAdmission::StaleCommandId => {
|
||||||
|
Err(WorkerCommandDisposition::StaleCommandId)
|
||||||
|
}
|
||||||
|
WorkerCommandAdmission::Conflict => Err(WorkerCommandDisposition::Conflict),
|
||||||
|
WorkerCommandAdmission::ExecutionGenerationMismatch => {
|
||||||
|
Err(WorkerCommandDisposition::StaleExecutionGeneration)
|
||||||
|
}
|
||||||
|
WorkerCommandAdmission::StateRevisionMismatch => {
|
||||||
|
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_command(
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
shared_state: &WorkerSharedState,
|
||||||
|
) -> Result<(), WorkerCommandDisposition> {
|
||||||
|
command_admission_disposition(shared_state.admit_command(envelope, kind, true))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_shutdown_command(
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
shared_state: &WorkerSharedState,
|
||||||
|
) -> Result<(), WorkerCommandDisposition> {
|
||||||
|
match shared_state.admit_command(envelope, WorkerCommandKind::Shutdown, false) {
|
||||||
|
WorkerCommandAdmission::Accepted | WorkerCommandAdmission::Retry => Ok(()),
|
||||||
|
admission => command_admission_disposition(admission),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn acknowledge_command(
|
||||||
|
working_event_tx: &broadcast::Sender<Event>,
|
||||||
|
shared_state: &WorkerSharedState,
|
||||||
|
command_id: u64,
|
||||||
|
command: WorkerCommandKind,
|
||||||
|
disposition: WorkerCommandDisposition,
|
||||||
|
) {
|
||||||
|
shared_state.complete_command(command_id, command, disposition);
|
||||||
|
let _ = working_event_tx.send(Event::CommandAcknowledged {
|
||||||
|
acknowledgement: WorkerCommandAcknowledgement {
|
||||||
|
command_id,
|
||||||
|
command,
|
||||||
|
disposition,
|
||||||
|
state: shared_state.snapshot(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject_invalid_command_state(
|
||||||
|
working_event_tx: &broadcast::Sender<Event>,
|
||||||
|
shared_state: &WorkerSharedState,
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
command: WorkerCommandKind,
|
||||||
|
) {
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
envelope.command_id,
|
||||||
|
command,
|
||||||
|
WorkerCommandDisposition::InvalidState,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_controller_state(
|
||||||
|
shared_state: &Arc<WorkerSharedState>,
|
||||||
|
runtime_dir: &RuntimeDir,
|
||||||
|
working_event_tx: &broadcast::Sender<Event>,
|
||||||
|
state: WorkerState,
|
||||||
|
) -> protocol::WorkerStateSnapshot {
|
||||||
|
let snapshot = shared_state.transition(state);
|
||||||
|
let _ = runtime_dir.write_status(shared_state).await;
|
||||||
|
let _ = working_event_tx.send(Event::WorkerState {
|
||||||
|
snapshot: snapshot.clone(),
|
||||||
|
});
|
||||||
|
snapshot
|
||||||
|
}
|
||||||
|
|
||||||
async fn set_controller_status(
|
async fn set_controller_status(
|
||||||
shared_state: &Arc<WorkerSharedState>,
|
shared_state: &Arc<WorkerSharedState>,
|
||||||
runtime_dir: &RuntimeDir,
|
runtime_dir: &RuntimeDir,
|
||||||
working_event_tx: &broadcast::Sender<Event>,
|
working_event_tx: &broadcast::Sender<Event>,
|
||||||
status: WorkerStatus,
|
status: WorkerStatus,
|
||||||
) {
|
) {
|
||||||
shared_state.set_status(status);
|
let state = match status {
|
||||||
let _ = runtime_dir.write_status(shared_state).await;
|
WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle,
|
||||||
let _ = working_event_tx.send(Event::Status { status });
|
WorkerStatus::Running => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||||
|
WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
|
||||||
|
};
|
||||||
|
set_controller_state(shared_state, runtime_dir, working_event_tx, state).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn finish_controller_run<C, St>(
|
async fn finish_controller_run<C, St>(
|
||||||
@@ -660,12 +749,24 @@ impl WorkerController {
|
|||||||
// === 4. Initial runtime files + WorkerSharedState + WorkerHandle +
|
// === 4. Initial runtime files + WorkerSharedState + WorkerHandle +
|
||||||
// SocketServer ===
|
// SocketServer ===
|
||||||
let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default();
|
let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default();
|
||||||
|
worker
|
||||||
|
.recover_unfinished_compaction()
|
||||||
|
.await
|
||||||
|
.map_err(|error| std::io::Error::other(error.to_string()))?;
|
||||||
let greeting = build_greeting(&worker);
|
let greeting = build_greeting(&worker);
|
||||||
let shared_state = Arc::new(WorkerSharedState::new(
|
let execution_generation = runtime_dir
|
||||||
|
.path()
|
||||||
|
.file_name()
|
||||||
|
.and_then(|name| name.to_str())
|
||||||
|
.and_then(|name| name.parse::<u64>().ok())
|
||||||
|
.filter(|generation| *generation > 0)
|
||||||
|
.unwrap_or(1);
|
||||||
|
let shared_state = Arc::new(WorkerSharedState::new_with_generation(
|
||||||
worker.manifest().worker.name.clone(),
|
worker.manifest().worker.name.clone(),
|
||||||
worker.segment_id(),
|
worker.segment_id(),
|
||||||
manifest_toml.clone(),
|
manifest_toml.clone(),
|
||||||
greeting,
|
greeting,
|
||||||
|
execution_generation,
|
||||||
));
|
));
|
||||||
if let Some(fs_for_view) = fs_for_view {
|
if let Some(fs_for_view) = fs_for_view {
|
||||||
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view));
|
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view));
|
||||||
@@ -1447,8 +1548,9 @@ async fn controller_loop<C, St>(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
loop {
|
let mut deferred_methods = VecDeque::new();
|
||||||
// Top-of-iteration: if an event handler staged a run, fire it
|
|
||||||
|
'controller: loop {
|
||||||
// here so the status flip → drive_turn → finish sequence lives
|
// here so the status flip → drive_turn → finish sequence lives
|
||||||
// in one place, regardless of which Method caused it.
|
// in one place, regardless of which Method caused it.
|
||||||
if let Some(run) = pending.take() {
|
if let Some(run) = pending.take() {
|
||||||
@@ -1599,9 +1701,13 @@ async fn controller_loop<C, St>(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let method = match method_rx.recv().await {
|
let method = if let Some(method) = deferred_methods.pop_front() {
|
||||||
Some(m) => m,
|
method
|
||||||
|
} else {
|
||||||
|
match method_rx.recv().await {
|
||||||
|
Some(method) => method,
|
||||||
None => break,
|
None => break,
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
match method {
|
match method {
|
||||||
@@ -1799,7 +1905,7 @@ async fn controller_loop<C, St>(
|
|||||||
expected_revision,
|
expected_revision,
|
||||||
expected_head_id,
|
expected_head_id,
|
||||||
} => {
|
} => {
|
||||||
if shared_state.get_status() != WorkerStatus::Idle {
|
if shared_state.catalog_status() != WorkerStatus::Idle {
|
||||||
let _ = working_event_tx.send(Event::Error {
|
let _ = working_event_tx.send(Event::Error {
|
||||||
code: ErrorCode::InvalidRequest,
|
code: ErrorCode::InvalidRequest,
|
||||||
message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(),
|
message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(),
|
||||||
@@ -1826,88 +1932,266 @@ async fn controller_loop<C, St>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Method::Resume => {
|
Method::Resume { command } => {
|
||||||
if shared_state.get_status() != WorkerStatus::Paused {
|
if let Err(disposition) =
|
||||||
let _ = working_event_tx.send(Event::Error {
|
validate_command(command, WorkerCommandKind::Resume, &shared_state)
|
||||||
code: ErrorCode::NotPaused,
|
{
|
||||||
message: "Worker is not paused".into(),
|
acknowledge_command(
|
||||||
});
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Resume,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
pending = Some(PendingRun::Resume);
|
if !matches!(
|
||||||
|
shared_state.snapshot().state,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused))
|
||||||
|
) {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Resume,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
set_controller_state(
|
||||||
Method::Cancel => match shared_state.get_status() {
|
|
||||||
WorkerStatus::Paused => match worker.cancel_paused_turn() {
|
|
||||||
Ok(()) => {
|
|
||||||
worker.clear_in_flight_events();
|
|
||||||
set_controller_status(
|
|
||||||
&shared_state,
|
&shared_state,
|
||||||
&runtime_dir,
|
&runtime_dir,
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
WorkerStatus::Idle,
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Resume,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
|
pending = Some(PendingRun::Resume);
|
||||||
|
}
|
||||||
|
|
||||||
|
Method::Cancel { command } => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Cancel, &shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
shared_state.snapshot().state,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused))
|
||||||
|
) {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
set_controller_state(
|
||||||
|
&shared_state,
|
||||||
|
&runtime_dir,
|
||||||
|
&working_event_tx,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
|
match worker.cancel_paused_turn() {
|
||||||
|
Ok(()) => {
|
||||||
|
worker.clear_in_flight_events();
|
||||||
|
set_controller_state(
|
||||||
|
&shared_state,
|
||||||
|
&runtime_dir,
|
||||||
|
&working_event_tx,
|
||||||
|
WorkerState::Idle,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let _ = working_event_tx.send(Event::Error {
|
set_controller_state(
|
||||||
code: worker_error_code(&error),
|
&shared_state,
|
||||||
message: error.to_string(),
|
&runtime_dir,
|
||||||
});
|
&working_event_tx,
|
||||||
}
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)),
|
||||||
},
|
)
|
||||||
WorkerStatus::Idle | WorkerStatus::Stopped => {
|
.await;
|
||||||
let _ = working_event_tx.send(Event::Error {
|
|
||||||
code: ErrorCode::NotRunning,
|
|
||||||
message: "Worker is not running".into(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
WorkerStatus::Running => {
|
|
||||||
// Running turns receive Cancel through drive_turn; this is
|
|
||||||
// only reachable across a defensive race window.
|
|
||||||
let _ = cancel_tx.try_send(());
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
Method::Pause => {
|
|
||||||
// Already paused → idempotent no-op. Otherwise the
|
|
||||||
// Worker is Idle (Running turns go through `drive_turn`,
|
|
||||||
// not this outer match), so there is nothing to pause.
|
|
||||||
if shared_state.get_status() != WorkerStatus::Paused {
|
|
||||||
let _ = working_event_tx.send(Event::Error {
|
|
||||||
code: ErrorCode::NotRunning,
|
|
||||||
message: "Worker is not running".into(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Method::Compact => match shared_state.get_status() {
|
|
||||||
WorkerStatus::Idle => {
|
|
||||||
if let Err(error) = worker.manual_compact().await {
|
|
||||||
let _ = working_event_tx.send(Event::Error {
|
let _ = working_event_tx.send(Event::Error {
|
||||||
code: worker_error_code(&error),
|
code: worker_error_code(&error),
|
||||||
message: error.to_string(),
|
message: error.to_string(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
WorkerStatus::Paused => {
|
|
||||||
let _ = working_event_tx.send(Event::Error {
|
|
||||||
code: ErrorCode::InvalidRequest,
|
|
||||||
message: "Cannot compact while the Worker is paused; resume or start a fresh turn first"
|
|
||||||
.into(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
WorkerStatus::Running | WorkerStatus::Stopped => {
|
|
||||||
let _ = working_event_tx.send(Event::Error {
|
|
||||||
code: ErrorCode::AlreadyRunning,
|
|
||||||
message:
|
|
||||||
"Worker is already executing a turn; compact can only run while idle"
|
|
||||||
.into(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
Method::ListRewindTargets => match shared_state.get_status() {
|
Method::Pause { command } => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Pause, &shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Method::Compact { command } => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Compact, &shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Compact,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !matches!(shared_state.snapshot().state, WorkerState::Idle) {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Compact,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
set_controller_state(
|
||||||
|
&shared_state,
|
||||||
|
&runtime_dir,
|
||||||
|
&working_event_tx,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Maintenance(
|
||||||
|
WorkerMaintenanceState::Compacting,
|
||||||
|
)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Compact,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
|
let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
|
||||||
|
let mut shutdown_after_compaction = false;
|
||||||
|
let result = {
|
||||||
|
let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx));
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut compact => break result,
|
||||||
|
method = method_rx.recv() => {
|
||||||
|
match method {
|
||||||
|
Some(Method::Cancel { command }) => {
|
||||||
|
if let Err(disposition) = validate_command(
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
&shared_state,
|
||||||
|
) {
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
|
let _ = cancel_tx.send(true);
|
||||||
|
}
|
||||||
|
Some(Method::Shutdown { command }) => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_shutdown_command(command, &shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
shutdown_after_compaction = true;
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
|
let _ = cancel_tx.send(true);
|
||||||
|
}
|
||||||
|
Some(method) => deferred_methods.push_back(method),
|
||||||
|
None => {
|
||||||
|
shutdown_after_compaction = true;
|
||||||
|
let _ = cancel_tx.send(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !matches!(
|
||||||
|
result,
|
||||||
|
Err(WorkerError::Store(_))
|
||||||
|
| Err(WorkerError::WorkerStore(_))
|
||||||
|
| Err(WorkerError::InvalidState(_))
|
||||||
|
) {
|
||||||
|
set_controller_state(
|
||||||
|
&shared_state,
|
||||||
|
&runtime_dir,
|
||||||
|
&working_event_tx,
|
||||||
|
WorkerState::Idle,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
if let Err(error) = result {
|
||||||
|
let _ = working_event_tx.send(Event::Error {
|
||||||
|
code: worker_error_code(&error),
|
||||||
|
message: error.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if shutdown_after_compaction {
|
||||||
|
let _ = working_event_tx.send(Event::Shutdown);
|
||||||
|
break 'controller;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Method::ListRewindTargets => match shared_state.catalog_status() {
|
||||||
WorkerStatus::Idle | WorkerStatus::Paused => {
|
WorkerStatus::Idle | WorkerStatus::Paused => {
|
||||||
emit_rewind_targets(&worker, &working_event_tx)
|
emit_rewind_targets(&worker, &working_event_tx)
|
||||||
}
|
}
|
||||||
@@ -1923,7 +2207,7 @@ async fn controller_loop<C, St>(
|
|||||||
Method::RewindTo {
|
Method::RewindTo {
|
||||||
target,
|
target,
|
||||||
expected_head_entries,
|
expected_head_entries,
|
||||||
} => match shared_state.get_status() {
|
} => match shared_state.catalog_status() {
|
||||||
WorkerStatus::Idle => {
|
WorkerStatus::Idle => {
|
||||||
if apply_rewind(
|
if apply_rewind(
|
||||||
&mut worker,
|
&mut worker,
|
||||||
@@ -1934,10 +2218,8 @@ async fn controller_loop<C, St>(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
worker.clear_in_flight_events();
|
worker.clear_in_flight_events();
|
||||||
shared_state.set_status(WorkerStatus::Idle);
|
let snapshot = shared_state.transition(WorkerState::Idle);
|
||||||
let _ = working_event_tx.send(Event::Status {
|
let _ = working_event_tx.send(Event::WorkerState { snapshot });
|
||||||
status: WorkerStatus::Idle,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
WorkerStatus::Paused => {
|
WorkerStatus::Paused => {
|
||||||
@@ -1956,7 +2238,26 @@ async fn controller_loop<C, St>(
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
Method::Shutdown => {
|
Method::Shutdown { command } => {
|
||||||
|
// Shutdown ignores the state-revision fence but remains bound to the
|
||||||
|
// current execution generation and command payload identity.
|
||||||
|
if let Err(disposition) = validate_shutdown_command(command, &shared_state) {
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
let _ = working_event_tx.send(Event::Shutdown);
|
let _ = working_event_tx.send(Event::Shutdown);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -2038,7 +2339,7 @@ async fn controller_loop<C, St>(
|
|||||||
// Auto-kick a turn if the Worker is idle so the
|
// Auto-kick a turn if the Worker is idle so the
|
||||||
// notification is not stranded. Matches the
|
// notification is not stranded. Matches the
|
||||||
// `Method::Notify` idle path.
|
// `Method::Notify` idle path.
|
||||||
if shared_state.get_status() == WorkerStatus::Idle {
|
if shared_state.catalog_status() == WorkerStatus::Idle {
|
||||||
pending = Some(PendingRun::RunForNotification {
|
pending = Some(PendingRun::RunForNotification {
|
||||||
invoke_kind: protocol::InvokeKind::WorkerEvent,
|
invoke_kind: protocol::InvokeKind::WorkerEvent,
|
||||||
notification_request_id: None,
|
notification_request_id: None,
|
||||||
@@ -2294,15 +2595,115 @@ where
|
|||||||
}
|
}
|
||||||
method = method_rx.recv(), if input_commit.is_none() => {
|
method = method_rx.recv(), if input_commit.is_none() => {
|
||||||
match method {
|
match method {
|
||||||
Some(Method::Cancel) => {
|
Some(Method::Cancel { command }) => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Cancel, shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
shared_state.snapshot().state,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||||
|
) {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
set_controller_state(
|
||||||
|
shared_state,
|
||||||
|
runtime_dir,
|
||||||
|
working_event_tx,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
let _ = cancel_tx.try_send(());
|
let _ = cancel_tx.try_send(());
|
||||||
}
|
}
|
||||||
Some(Method::Pause) => {
|
Some(Method::Pause { command }) => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Pause, shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
shared_state.snapshot().state,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||||
|
) {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
pause_requested = true;
|
pause_requested = true;
|
||||||
|
set_controller_state(
|
||||||
|
shared_state,
|
||||||
|
runtime_dir,
|
||||||
|
working_event_tx,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Pausing)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
let _ = pause_tx.try_send(());
|
let _ = pause_tx.try_send(());
|
||||||
}
|
}
|
||||||
Some(Method::Shutdown) => {
|
Some(Method::Shutdown { command }) => {
|
||||||
|
if let Err(disposition) = validate_shutdown_command(command, shared_state) {
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
shutdown_requested = true;
|
shutdown_requested = true;
|
||||||
|
set_controller_state(
|
||||||
|
shared_state,
|
||||||
|
runtime_dir,
|
||||||
|
working_event_tx,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
let _ = cancel_tx.try_send(());
|
let _ = cancel_tx.try_send(());
|
||||||
}
|
}
|
||||||
Some(Method::Submit {
|
Some(Method::Submit {
|
||||||
@@ -2368,7 +2769,27 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some(Method::Resume | Method::ContinuePending { .. }) => {
|
Some(Method::Resume { command }) => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Resume, shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Resume,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Resume,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(Method::ContinuePending { .. }) => {
|
||||||
let _ = working_event_tx.send(Event::Error {
|
let _ = working_event_tx.send(Event::Error {
|
||||||
code: ErrorCode::AlreadyRunning,
|
code: ErrorCode::AlreadyRunning,
|
||||||
message: "Worker is already executing a turn".into(),
|
message: "Worker is already executing a turn".into(),
|
||||||
@@ -2408,7 +2829,27 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => {
|
Some(Method::Compact { command }) => {
|
||||||
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Compact, shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Compact,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
reject_invalid_command_state(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Compact,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(Method::ListRewindTargets | Method::RewindTo { .. }) => {
|
||||||
let _ = working_event_tx.send(Event::Error {
|
let _ = working_event_tx.send(Event::Error {
|
||||||
code: ErrorCode::AlreadyRunning,
|
code: ErrorCode::AlreadyRunning,
|
||||||
message: "Worker is already executing a turn; rewind/compact can only run while idle or paused"
|
message: "Worker is already executing a turn; rewind/compact can only run while idle or paused"
|
||||||
@@ -2511,7 +2952,7 @@ where
|
|||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
let _ = cancel_tx.try_send(());
|
let _ = cancel_tx.try_send(());
|
||||||
shared_state.set_status(WorkerStatus::Idle);
|
shared_state.transition(WorkerState::Idle);
|
||||||
return (WorkerStatus::Idle, false, false);
|
return (WorkerStatus::Idle, false, false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2887,7 +3328,7 @@ mod tests {
|
|||||||
context_window: 200_000,
|
context_window: 200_000,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
@@ -2943,9 +3384,17 @@ mod tests {
|
|||||||
async fn pause_waits_for_run_boundary_and_uses_safe_pause_channel() {
|
async fn pause_waits_for_run_boundary_and_uses_safe_pause_channel() {
|
||||||
let mut env = make_env().await;
|
let mut env = make_env().await;
|
||||||
let method_tx = env._method_tx.clone();
|
let method_tx = env._method_tx.clone();
|
||||||
|
env.shared_state
|
||||||
|
.transition(WorkerState::Busy(WorkerBusyState::Run(
|
||||||
|
WorkerRunState::Running,
|
||||||
|
)));
|
||||||
|
let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot());
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
method_tx.send(Method::Pause).await.expect("send pause");
|
method_tx
|
||||||
|
.send(Method::Pause { command })
|
||||||
|
.await
|
||||||
|
.expect("send pause");
|
||||||
});
|
});
|
||||||
|
|
||||||
let worker_future = async {
|
let worker_future = async {
|
||||||
@@ -3218,8 +3667,13 @@ mod tests {
|
|||||||
async fn compact_method_is_rejected_while_running() {
|
async fn compact_method_is_rejected_while_running() {
|
||||||
let mut env = make_env().await;
|
let mut env = make_env().await;
|
||||||
let mut events = env.working_event_tx.subscribe();
|
let mut events = env.working_event_tx.subscribe();
|
||||||
|
env.shared_state
|
||||||
|
.transition(WorkerState::Busy(WorkerBusyState::Run(
|
||||||
|
WorkerRunState::Running,
|
||||||
|
)));
|
||||||
|
let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot());
|
||||||
env._method_tx
|
env._method_tx
|
||||||
.send(Method::Compact)
|
.send(Method::Compact { command })
|
||||||
.await
|
.await
|
||||||
.expect("send compact");
|
.expect("send compact");
|
||||||
|
|
||||||
@@ -3252,14 +3706,113 @@ mod tests {
|
|||||||
.expect("event timeout")
|
.expect("event timeout")
|
||||||
.expect("event");
|
.expect("event");
|
||||||
match event {
|
match event {
|
||||||
Event::Error { code, message } => {
|
Event::CommandAcknowledged { acknowledgement } => {
|
||||||
assert_eq!(code, ErrorCode::AlreadyRunning);
|
assert_eq!(acknowledgement.command, WorkerCommandKind::Compact);
|
||||||
assert!(message.contains("compact"), "got message: {message}");
|
assert_eq!(
|
||||||
|
acknowledgement.disposition,
|
||||||
|
WorkerCommandDisposition::InvalidState
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
acknowledgement.state.state,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running))
|
||||||
|
));
|
||||||
}
|
}
|
||||||
other => panic!("expected compact rejection error, got {other:?}"),
|
other => panic!("expected compact rejection acknowledgement, got {other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn command_admission_rejects_stale_generation_revision_and_order() {
|
||||||
|
let shared = WorkerSharedState::new_with_generation(
|
||||||
|
"worker".into(),
|
||||||
|
session_store::new_segment_id(),
|
||||||
|
String::new(),
|
||||||
|
protocol::Greeting {
|
||||||
|
worker_name: "worker".into(),
|
||||||
|
cwd: "/tmp".into(),
|
||||||
|
provider: "test".into(),
|
||||||
|
model: "test".into(),
|
||||||
|
scope_summary: String::new(),
|
||||||
|
tools: Vec::new(),
|
||||||
|
context_window: 1,
|
||||||
|
context_tokens: 0,
|
||||||
|
},
|
||||||
|
9,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
validate_command(
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 8,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
&shared,
|
||||||
|
),
|
||||||
|
Err(WorkerCommandDisposition::StaleExecutionGeneration)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
validate_command(
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 2,
|
||||||
|
expected_execution_generation: 9,
|
||||||
|
expected_worker_state_revision: 1,
|
||||||
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
&shared,
|
||||||
|
),
|
||||||
|
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
validate_command(
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 9,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
&shared,
|
||||||
|
)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
validate_command(
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 9,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
&shared,
|
||||||
|
),
|
||||||
|
Err(WorkerCommandDisposition::StaleCommandId)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
validate_command(
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 9,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
},
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
&shared,
|
||||||
|
),
|
||||||
|
Err(WorkerCommandDisposition::Conflict)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
validate_command(
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 2,
|
||||||
|
expected_execution_generation: 9,
|
||||||
|
expected_worker_state_revision: 1,
|
||||||
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
&shared,
|
||||||
|
)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn controller_shutdown_orders_child_cleanup_before_workdir_close() {
|
fn controller_shutdown_orders_child_cleanup_before_workdir_close() {
|
||||||
let source = include_str!("controller.rs");
|
let source = include_str!("controller.rs");
|
||||||
|
|||||||
@@ -779,10 +779,10 @@ async fn probe_socket(socket_path: &Path) -> LiveInfo {
|
|||||||
loop {
|
loop {
|
||||||
match tokio::time::timeout(PROBE_TIMEOUT, reader.next::<Event>()).await {
|
match tokio::time::timeout(PROBE_TIMEOUT, reader.next::<Event>()).await {
|
||||||
Ok(Ok(Some(Event::Snapshot {
|
Ok(Ok(Some(Event::Snapshot {
|
||||||
status: snapshot_status,
|
state: snapshot_state,
|
||||||
..
|
..
|
||||||
}))) => {
|
}))) => {
|
||||||
status = Some(snapshot_status);
|
status = Some(snapshot_state.catalog_status());
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Ok(Ok(Some(Event::Alert(_)))) => continue,
|
Ok(Ok(Some(Event::Alert(_)))) => continue,
|
||||||
@@ -1507,7 +1507,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
@@ -1543,7 +1543,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
@@ -1638,7 +1638,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
@@ -1665,7 +1665,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
@@ -1773,7 +1773,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Paused,
|
state: WorkerStatus::Paused.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
@@ -1827,7 +1827,7 @@ mod tests {
|
|||||||
context_window: 0,
|
context_window: 0,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -295,6 +295,38 @@ impl InternalWorkerSessionStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn send_internal_worker_state(
|
||||||
|
event_tx: &broadcast::Sender<Event>,
|
||||||
|
state_revision: &std::sync::atomic::AtomicU64,
|
||||||
|
status: InternalWorkerSessionStatus,
|
||||||
|
) {
|
||||||
|
let state = match status {
|
||||||
|
InternalWorkerSessionStatus::Idle
|
||||||
|
| InternalWorkerSessionStatus::Stopped
|
||||||
|
| InternalWorkerSessionStatus::Failed => protocol::WorkerState::Idle,
|
||||||
|
InternalWorkerSessionStatus::Paused => protocol::WorkerState::Busy(
|
||||||
|
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Paused),
|
||||||
|
),
|
||||||
|
InternalWorkerSessionStatus::Running => protocol::WorkerState::Busy(
|
||||||
|
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running),
|
||||||
|
),
|
||||||
|
InternalWorkerSessionStatus::Stopping => protocol::WorkerState::Busy(
|
||||||
|
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Cancelling),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
let revision = state_revision
|
||||||
|
.fetch_add(1, std::sync::atomic::Ordering::AcqRel)
|
||||||
|
.saturating_add(1);
|
||||||
|
let _ = event_tx.send(Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision,
|
||||||
|
last_command_id: 0,
|
||||||
|
state,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
fn classify_internal_turn_result(
|
fn classify_internal_turn_result(
|
||||||
result: Result<WorkerRunResult, WorkerError>,
|
result: Result<WorkerRunResult, WorkerError>,
|
||||||
) -> (InternalWorkerSessionStatus, Option<String>) {
|
) -> (InternalWorkerSessionStatus, Option<String>) {
|
||||||
@@ -351,6 +383,7 @@ pub(crate) struct InternalWorkerSessionSnapshot {
|
|||||||
pub(crate) struct InternalWorkerSessionHandle {
|
pub(crate) struct InternalWorkerSessionHandle {
|
||||||
command_tx: tokio::sync::mpsc::Sender<InternalWorkerSessionCommand>,
|
command_tx: tokio::sync::mpsc::Sender<InternalWorkerSessionCommand>,
|
||||||
status: Arc<std::sync::atomic::AtomicU8>,
|
status: Arc<std::sync::atomic::AtomicU8>,
|
||||||
|
state_revision: Arc<std::sync::atomic::AtomicU64>,
|
||||||
store: EphemeralSessionStore,
|
store: EphemeralSessionStore,
|
||||||
session_id: SessionId,
|
session_id: SessionId,
|
||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
@@ -400,6 +433,10 @@ impl InternalWorkerSessionHandle {
|
|||||||
self.in_flight.text_delta(block_id, text.to_owned());
|
self.in_flight.text_delta(block_id, text.to_owned());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn emit_worker_state(&self, status: InternalWorkerSessionStatus) {
|
||||||
|
send_internal_worker_state(&self.event_tx, &self.state_revision, status);
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn protocol_snapshot(&self) -> InternalWorkerSessionSnapshot {
|
pub(crate) fn protocol_snapshot(&self) -> InternalWorkerSessionSnapshot {
|
||||||
let (entries, in_flight) = {
|
let (entries, in_flight) = {
|
||||||
let guard = self.in_flight.snapshot_guard();
|
let guard = self.in_flight.snapshot_guard();
|
||||||
@@ -473,9 +510,7 @@ impl InternalWorkerSessionHandle {
|
|||||||
});
|
});
|
||||||
return Err(InternalWorkerSessionError::Unavailable);
|
return Err(InternalWorkerSessionError::Unavailable);
|
||||||
}
|
}
|
||||||
let _ = self.event_tx.send(Event::Status {
|
self.emit_worker_state(InternalWorkerSessionStatus::Running);
|
||||||
status: WorkerStatus::Running,
|
|
||||||
});
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -770,11 +805,13 @@ pub(crate) async fn prepare_internal_worker_session(
|
|||||||
let status = Arc::new(std::sync::atomic::AtomicU8::new(
|
let status = Arc::new(std::sync::atomic::AtomicU8::new(
|
||||||
InternalWorkerSessionStatus::Idle.encode(),
|
InternalWorkerSessionStatus::Idle.encode(),
|
||||||
));
|
));
|
||||||
|
let state_revision = Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||||
let state_changed = Arc::new(tokio::sync::Notify::new());
|
let state_changed = Arc::new(tokio::sync::Notify::new());
|
||||||
let last_error = Arc::new(Mutex::new(None));
|
let last_error = Arc::new(Mutex::new(None));
|
||||||
let handle = InternalWorkerSessionHandle {
|
let handle = InternalWorkerSessionHandle {
|
||||||
command_tx,
|
command_tx,
|
||||||
status: status.clone(),
|
status: status.clone(),
|
||||||
|
state_revision: state_revision.clone(),
|
||||||
store,
|
store,
|
||||||
session_id,
|
session_id,
|
||||||
segment_id,
|
segment_id,
|
||||||
@@ -810,19 +847,11 @@ pub(crate) async fn prepare_internal_worker_session(
|
|||||||
message,
|
message,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let protocol_status = match turn_status {
|
send_internal_worker_state(
|
||||||
InternalWorkerSessionStatus::Idle => WorkerStatus::Idle,
|
&event_tx,
|
||||||
InternalWorkerSessionStatus::Paused => WorkerStatus::Paused,
|
&state_revision,
|
||||||
InternalWorkerSessionStatus::Stopped
|
turn_status,
|
||||||
| InternalWorkerSessionStatus::Failed => WorkerStatus::Stopped,
|
);
|
||||||
InternalWorkerSessionStatus::Running
|
|
||||||
| InternalWorkerSessionStatus::Stopping => {
|
|
||||||
unreachable!("run completion cannot remain active")
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let _ = event_tx.send(Event::Status {
|
|
||||||
status: protocol_status,
|
|
||||||
});
|
|
||||||
if let Some(callback) = &on_turn_end {
|
if let Some(callback) = &on_turn_end {
|
||||||
callback(turn_status);
|
callback(turn_status);
|
||||||
}
|
}
|
||||||
@@ -864,9 +893,11 @@ pub(crate) async fn prepare_internal_worker_session(
|
|||||||
InternalWorkerSessionStatus::Stopped.encode(),
|
InternalWorkerSessionStatus::Stopped.encode(),
|
||||||
std::sync::atomic::Ordering::Release,
|
std::sync::atomic::Ordering::Release,
|
||||||
);
|
);
|
||||||
let _ = event_tx.send(Event::Status {
|
send_internal_worker_state(
|
||||||
status: WorkerStatus::Stopped,
|
&event_tx,
|
||||||
});
|
&state_revision,
|
||||||
|
InternalWorkerSessionStatus::Stopped,
|
||||||
|
);
|
||||||
let _ = event_tx.send(Event::Shutdown);
|
let _ = event_tx.send(Event::Shutdown);
|
||||||
state_changed.notify_waiters();
|
state_changed.notify_waiters();
|
||||||
if let Some(done) = stop_done {
|
if let Some(done) = stop_done {
|
||||||
@@ -1118,6 +1149,7 @@ pub(crate) fn test_internal_worker_session(
|
|||||||
status: Arc::new(std::sync::atomic::AtomicU8::new(
|
status: Arc::new(std::sync::atomic::AtomicU8::new(
|
||||||
InternalWorkerSessionStatus::Idle.encode(),
|
InternalWorkerSessionStatus::Idle.encode(),
|
||||||
)),
|
)),
|
||||||
|
state_revision: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||||
store,
|
store,
|
||||||
session_id,
|
session_id,
|
||||||
segment_id,
|
segment_id,
|
||||||
|
|||||||
@@ -197,7 +197,6 @@ pub fn default_base() -> Result<PathBuf, io::Error> {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::shared_state::WorkerSharedState;
|
use crate::shared_state::WorkerSharedState;
|
||||||
use protocol::WorkerStatus;
|
|
||||||
|
|
||||||
fn test_state() -> WorkerSharedState {
|
fn test_state() -> WorkerSharedState {
|
||||||
WorkerSharedState::new(
|
WorkerSharedState::new(
|
||||||
@@ -247,7 +246,9 @@ mod tests {
|
|||||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
|
|
||||||
state.set_status(WorkerStatus::Running);
|
state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)));
|
||||||
rt.write_status(&state).await.unwrap();
|
rt.write_status(&state).await.unwrap();
|
||||||
|
|
||||||
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
|
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
|
||||||
|
|||||||
@@ -1,28 +1,49 @@
|
|||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::collections::VecDeque;
|
||||||
use std::sync::{OnceLock, RwLock};
|
use std::sync::{
|
||||||
|
OnceLock, RwLock,
|
||||||
|
atomic::{AtomicBool, Ordering},
|
||||||
|
};
|
||||||
|
|
||||||
use protocol::WorkerStatus;
|
use protocol::{
|
||||||
|
WorkerBusyState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind,
|
||||||
|
WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus,
|
||||||
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use session_store::SegmentId;
|
use session_store::SegmentId;
|
||||||
|
|
||||||
use crate::fs_view::WorkerFsView;
|
use crate::fs_view::WorkerFsView;
|
||||||
|
|
||||||
|
const COMPLETED_COMMAND_RETENTION: usize = 256;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
struct AcceptedWorkerCommand {
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
disposition: Option<WorkerCommandDisposition>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum WorkerCommandAdmission {
|
||||||
|
Accepted,
|
||||||
|
Retry,
|
||||||
|
Conflict,
|
||||||
|
StaleCommandId,
|
||||||
|
ExecutionGenerationMismatch,
|
||||||
|
StateRevisionMismatch,
|
||||||
|
}
|
||||||
|
|
||||||
/// Shared state between WorkerController and runtime directory.
|
/// Shared state between WorkerController and runtime directory.
|
||||||
///
|
///
|
||||||
/// Controller updates this in-memory; RuntimeDir writes the status
|
/// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime
|
||||||
/// snapshot to disk. Wrapped in `Arc` for sharing.
|
/// catalog status remains a separate lifecycle projection because `Stopped`
|
||||||
///
|
/// describes the execution handle rather than a live controller state.
|
||||||
/// History and typed user-segment mirrors used to live here so the
|
|
||||||
/// IPC layer could answer `Method::GetHistory`. Those reads now go
|
|
||||||
/// directly through the session-log sink (`Event::Snapshot` +
|
|
||||||
/// live events), so this struct holds only status, identity,
|
|
||||||
/// greeting, and filesystem completion lookup hubs.
|
|
||||||
pub struct WorkerSharedState {
|
pub struct WorkerSharedState {
|
||||||
pub worker_name: String,
|
pub worker_name: String,
|
||||||
pub segment_id: SegmentId,
|
pub segment_id: SegmentId,
|
||||||
pub manifest_toml: String,
|
pub manifest_toml: String,
|
||||||
pub greeting: protocol::Greeting,
|
pub greeting: protocol::Greeting,
|
||||||
pub status: RwLock<WorkerStatus>,
|
state: RwLock<WorkerStateSnapshot>,
|
||||||
|
accepted_commands: RwLock<VecDeque<AcceptedWorkerCommand>>,
|
||||||
/// Worker-from-the-inside view of the filesystem. Set once in
|
/// Worker-from-the-inside view of the filesystem. Set once in
|
||||||
/// `WorkerController::start` after the local WorkdirSession provider is
|
/// `WorkerController::start` after the local WorkdirSession provider is
|
||||||
/// materialised, and read from the IPC server layer to answer
|
/// materialised, and read from the IPC server layer to answer
|
||||||
@@ -38,13 +59,24 @@ impl WorkerSharedState {
|
|||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
manifest_toml: String,
|
manifest_toml: String,
|
||||||
greeting: protocol::Greeting,
|
greeting: protocol::Greeting,
|
||||||
|
) -> Self {
|
||||||
|
Self::new_with_generation(worker_name, segment_id, manifest_toml, greeting, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new_with_generation(
|
||||||
|
worker_name: String,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
manifest_toml: String,
|
||||||
|
greeting: protocol::Greeting,
|
||||||
|
execution_generation: u64,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
worker_name,
|
worker_name,
|
||||||
segment_id,
|
segment_id,
|
||||||
manifest_toml,
|
manifest_toml,
|
||||||
greeting,
|
greeting,
|
||||||
status: RwLock::new(WorkerStatus::Idle),
|
state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)),
|
||||||
|
accepted_commands: RwLock::new(VecDeque::new()),
|
||||||
fs_view: OnceLock::new(),
|
fs_view: OnceLock::new(),
|
||||||
flow_transition_enabled: AtomicBool::new(false),
|
flow_transition_enabled: AtomicBool::new(false),
|
||||||
}
|
}
|
||||||
@@ -70,21 +102,140 @@ impl WorkerSharedState {
|
|||||||
self.flow_transition_enabled.load(Ordering::Acquire)
|
self.flow_transition_enabled.load(Ordering::Acquire)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_status(&self, status: WorkerStatus) {
|
pub fn transition(&self, state: WorkerState) -> WorkerStateSnapshot {
|
||||||
if let Ok(mut s) = self.status.write() {
|
let mut snapshot = self
|
||||||
*s = status;
|
.state
|
||||||
|
.write()
|
||||||
|
.expect("worker state lock poisoned; refusing an inferred fallback state");
|
||||||
|
if snapshot.state != state {
|
||||||
|
snapshot.revision = snapshot.revision.saturating_add(1);
|
||||||
|
snapshot.state = state;
|
||||||
|
}
|
||||||
|
snapshot.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn admit_command(
|
||||||
|
&self,
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
require_state_revision: bool,
|
||||||
|
) -> WorkerCommandAdmission {
|
||||||
|
let mut snapshot = self
|
||||||
|
.state
|
||||||
|
.write()
|
||||||
|
.expect("worker state lock poisoned; refusing command admission");
|
||||||
|
let mut accepted = self
|
||||||
|
.accepted_commands
|
||||||
|
.write()
|
||||||
|
.expect("worker command ledger lock poisoned; refusing command admission");
|
||||||
|
if let Some(existing) = accepted
|
||||||
|
.iter()
|
||||||
|
.find(|accepted| accepted.envelope.command_id == envelope.command_id)
|
||||||
|
{
|
||||||
|
return if existing.envelope == envelope && existing.kind == kind {
|
||||||
|
WorkerCommandAdmission::Retry
|
||||||
|
} else {
|
||||||
|
WorkerCommandAdmission::Conflict
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if envelope.expected_execution_generation != snapshot.execution_generation {
|
||||||
|
return WorkerCommandAdmission::ExecutionGenerationMismatch;
|
||||||
|
}
|
||||||
|
if require_state_revision && envelope.expected_worker_state_revision != snapshot.revision {
|
||||||
|
return WorkerCommandAdmission::StateRevisionMismatch;
|
||||||
|
}
|
||||||
|
if envelope.command_id <= snapshot.last_command_id {
|
||||||
|
return WorkerCommandAdmission::StaleCommandId;
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.last_command_id = envelope.command_id;
|
||||||
|
snapshot.revision = snapshot.revision.saturating_add(1);
|
||||||
|
accepted.push_back(AcceptedWorkerCommand {
|
||||||
|
envelope,
|
||||||
|
kind,
|
||||||
|
disposition: None,
|
||||||
|
});
|
||||||
|
WorkerCommandAdmission::Accepted
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn complete_command(
|
||||||
|
&self,
|
||||||
|
command_id: u64,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
disposition: WorkerCommandDisposition,
|
||||||
|
) {
|
||||||
|
if !matches!(
|
||||||
|
disposition,
|
||||||
|
WorkerCommandDisposition::Accepted | WorkerCommandDisposition::InvalidState
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut accepted = self
|
||||||
|
.accepted_commands
|
||||||
|
.write()
|
||||||
|
.expect("worker command ledger lock poisoned; refusing command completion");
|
||||||
|
if let Some(command) = accepted
|
||||||
|
.iter_mut()
|
||||||
|
.find(|command| command.envelope.command_id == command_id && command.kind == kind)
|
||||||
|
{
|
||||||
|
command.disposition.get_or_insert(disposition);
|
||||||
|
}
|
||||||
|
while accepted
|
||||||
|
.iter()
|
||||||
|
.filter(|command| command.disposition.is_some())
|
||||||
|
.count()
|
||||||
|
> COMPLETED_COMMAND_RETENTION
|
||||||
|
{
|
||||||
|
let Some(index) = accepted
|
||||||
|
.iter()
|
||||||
|
.position(|command| command.disposition.is_some())
|
||||||
|
else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
accepted.remove(index);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_status(&self) -> WorkerStatus {
|
#[cfg(test)]
|
||||||
self.status.read().map(|s| *s).unwrap_or(WorkerStatus::Idle)
|
pub(crate) fn command_result(
|
||||||
|
&self,
|
||||||
|
command_id: u64,
|
||||||
|
) -> Option<Option<WorkerCommandDisposition>> {
|
||||||
|
self.accepted_commands
|
||||||
|
.read()
|
||||||
|
.expect("worker command ledger lock poisoned")
|
||||||
|
.iter()
|
||||||
|
.find(|command| command.envelope.command_id == command_id)
|
||||||
|
.map(|command| command.disposition)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Serialize status as JSON.
|
pub fn snapshot(&self) -> WorkerStateSnapshot {
|
||||||
|
self.state
|
||||||
|
.read()
|
||||||
|
.expect("worker state lock poisoned; refusing an inferred fallback state")
|
||||||
|
.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runtime catalog projection. This must not be used as live command
|
||||||
|
/// admission authority.
|
||||||
|
pub fn catalog_status(&self) -> WorkerStatus {
|
||||||
|
match self.snapshot().state {
|
||||||
|
WorkerState::Idle => WorkerStatus::Idle,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused,
|
||||||
|
WorkerState::Busy(WorkerBusyState::Run(_))
|
||||||
|
| WorkerState::Busy(WorkerBusyState::Maintenance(WorkerMaintenanceState::Compacting)) => {
|
||||||
|
WorkerStatus::Running
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serialize the runtime-directory lifecycle projection as JSON while
|
||||||
|
/// retaining the full state snapshot for diagnostics and reconnects.
|
||||||
pub fn status_json(&self) -> String {
|
pub fn status_json(&self) -> String {
|
||||||
let status = self.get_status();
|
let snapshot = self.snapshot();
|
||||||
json!({
|
json!({
|
||||||
"state": status,
|
"state": self.catalog_status(),
|
||||||
|
"worker_state": snapshot,
|
||||||
"segment_id": self.segment_id.to_string(),
|
"segment_id": self.segment_id.to_string(),
|
||||||
"worker_name": self.worker_name,
|
"worker_name": self.worker_name,
|
||||||
})
|
})
|
||||||
@@ -97,11 +248,12 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
fn test_state() -> WorkerSharedState {
|
fn test_state() -> WorkerSharedState {
|
||||||
WorkerSharedState::new(
|
WorkerSharedState::new_with_generation(
|
||||||
"test-worker".into(),
|
"test-worker".into(),
|
||||||
session_store::new_segment_id(),
|
session_store::new_segment_id(),
|
||||||
"[engine]\nname = \"test-worker\"".into(),
|
"[engine]\nname = \"test-worker\"".into(),
|
||||||
test_greeting(),
|
test_greeting(),
|
||||||
|
7,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -119,36 +271,82 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn initial_status_is_idle() {
|
fn initial_snapshot_is_idle() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
assert_eq!(state.get_status(), WorkerStatus::Idle);
|
assert_eq!(state.snapshot(), WorkerStateSnapshot::initial(7));
|
||||||
|
assert_eq!(state.catalog_status(), WorkerStatus::Idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn set_and_get_status() {
|
fn transitions_increment_revision_only_when_state_changes() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
state.set_status(WorkerStatus::Running);
|
let running = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running));
|
||||||
assert_eq!(state.get_status(), WorkerStatus::Running);
|
let snapshot = state.transition(running.clone());
|
||||||
state.set_status(WorkerStatus::Paused);
|
assert_eq!(snapshot.revision, 1);
|
||||||
assert_eq!(state.get_status(), WorkerStatus::Paused);
|
assert_eq!(snapshot.state, running);
|
||||||
|
assert_eq!(state.transition(running).revision, 1);
|
||||||
|
|
||||||
|
let paused = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused));
|
||||||
|
let snapshot = state.transition(paused.clone());
|
||||||
|
assert_eq!(snapshot.revision, 2);
|
||||||
|
assert_eq!(snapshot.state, paused);
|
||||||
|
assert_eq!(state.catalog_status(), WorkerStatus::Paused);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn status_json_contains_fields() {
|
fn accepted_command_identity_advances_revision_and_detects_reuse_conflicts() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
let json = state.status_json();
|
let envelope = WorkerCommandEnvelope {
|
||||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
command_id: 9,
|
||||||
assert_eq!(parsed["state"], "idle");
|
expected_execution_generation: 7,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
state.admit_command(envelope, WorkerCommandKind::Pause, true),
|
||||||
|
WorkerCommandAdmission::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.snapshot(),
|
||||||
|
WorkerStateSnapshot {
|
||||||
|
execution_generation: 7,
|
||||||
|
revision: 1,
|
||||||
|
last_command_id: 9,
|
||||||
|
state: WorkerState::Idle,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(state.command_result(9), Some(None));
|
||||||
|
state.complete_command(
|
||||||
|
9,
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.command_result(9),
|
||||||
|
Some(Some(WorkerCommandDisposition::Accepted))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.admit_command(envelope, WorkerCommandKind::Pause, true),
|
||||||
|
WorkerCommandAdmission::Retry
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.admit_command(envelope, WorkerCommandKind::Cancel, true),
|
||||||
|
WorkerCommandAdmission::Conflict
|
||||||
|
);
|
||||||
|
assert_eq!(state.snapshot().revision, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn status_json_contains_full_snapshot_and_catalog_projection() {
|
||||||
|
let state = test_state();
|
||||||
|
state.transition(WorkerState::Busy(WorkerBusyState::Maintenance(
|
||||||
|
WorkerMaintenanceState::Compacting,
|
||||||
|
)));
|
||||||
|
let parsed: serde_json::Value = serde_json::from_str(&state.status_json()).unwrap();
|
||||||
|
assert_eq!(parsed["state"], "running");
|
||||||
|
assert_eq!(parsed["worker_state"]["execution_generation"], 7);
|
||||||
|
assert_eq!(parsed["worker_state"]["revision"], 1);
|
||||||
|
assert_eq!(parsed["worker_state"]["state"]["kind"], "busy");
|
||||||
assert_eq!(parsed["worker_name"], "test-worker");
|
assert_eq!(parsed["worker_name"], "test-worker");
|
||||||
assert!(parsed["segment_id"].is_string());
|
assert!(parsed["segment_id"].is_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn status_json_reflects_changes() {
|
|
||||||
let state = test_state();
|
|
||||||
state.set_status(WorkerStatus::Running);
|
|
||||||
let json = state.status_json();
|
|
||||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
|
||||||
assert_eq!(parsed["state"], "running");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ mod tests {
|
|||||||
context_window: 200_000,
|
context_window: 200_000,
|
||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: WorkerStatus::Idle,
|
state: WorkerStatus::Idle.into(),
|
||||||
in_flight: Default::default(),
|
in_flight: Default::default(),
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
}
|
}
|
||||||
@@ -137,10 +137,16 @@ mod tests {
|
|||||||
],
|
],
|
||||||
);
|
);
|
||||||
|
|
||||||
connect_and_send(&socket, &Method::Shutdown).await.unwrap();
|
let method = Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::for_snapshot(
|
||||||
|
1,
|
||||||
|
&protocol::WorkerStateSnapshot::initial(1),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
connect_and_send(&socket, &method).await.unwrap();
|
||||||
|
|
||||||
let method = received.await.unwrap().expect("expected method");
|
let method = received.await.unwrap().expect("expected method");
|
||||||
assert!(matches!(method, Method::Shutdown));
|
assert!(matches!(method, Method::Shutdown { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
+186
-20
@@ -4571,15 +4571,6 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn persist_and_send_compact_done(
|
|
||||||
&mut self,
|
|
||||||
lifecycle: CompactionLifecycle,
|
|
||||||
) -> Result<(), WorkerError> {
|
|
||||||
self.persist_compaction_lifecycle(&lifecycle)?;
|
|
||||||
self.send_event(Event::CompactDone { lifecycle });
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn persist_and_send_compact_failed(
|
fn persist_and_send_compact_failed(
|
||||||
&mut self,
|
&mut self,
|
||||||
lifecycle: CompactionLifecycle,
|
lifecycle: CompactionLifecycle,
|
||||||
@@ -4724,7 +4715,97 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
Ok(rewrite_guard)
|
Ok(rewrite_guard)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Terminalize and clean up any compaction that was left active by the
|
||||||
|
/// previous controller generation. This runs before the restored
|
||||||
|
/// controller publishes its first Idle state.
|
||||||
|
pub async fn recover_unfinished_compaction(&mut self) -> Result<(), WorkerError> {
|
||||||
|
let (entries, _) = self.sink.subscribe_with_snapshot();
|
||||||
|
let latest_payload = entries.iter().rev().find_map(|entry| match entry {
|
||||||
|
LogEntry::Extension {
|
||||||
|
domain, payload, ..
|
||||||
|
} if domain == COMPACTION_EXTENSION_DOMAIN => Some(payload.clone()),
|
||||||
|
_ => None,
|
||||||
|
});
|
||||||
|
let Some(payload) = latest_payload else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct CompactionLifecycleWire {
|
||||||
|
schema_version: u32,
|
||||||
|
compaction_id: String,
|
||||||
|
revision: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
internal_worker: Option<protocol::InternalWorkerRef>,
|
||||||
|
state: CompactionLifecycleState,
|
||||||
|
started_at_ms: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
ended_at_ms: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
summary: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
error: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
new_segment_id: Option<String>,
|
||||||
|
}
|
||||||
|
let wire: CompactionLifecycleWire = serde_json::from_value(payload).map_err(|error| {
|
||||||
|
WorkerError::InvalidState(format!("decode compaction lifecycle: {error}"))
|
||||||
|
})?;
|
||||||
|
if !matches!(wire.schema_version, 2 | 3) {
|
||||||
|
return Err(WorkerError::InvalidState(format!(
|
||||||
|
"unsupported compaction lifecycle schema version {}",
|
||||||
|
wire.schema_version
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let mut lifecycle = CompactionLifecycle {
|
||||||
|
schema_version: wire.schema_version,
|
||||||
|
compaction_id: wire.compaction_id,
|
||||||
|
revision: wire.revision,
|
||||||
|
internal_worker: wire.internal_worker,
|
||||||
|
state: wire.state,
|
||||||
|
started_at_ms: wire.started_at_ms,
|
||||||
|
ended_at_ms: wire.ended_at_ms,
|
||||||
|
summary: wire.summary,
|
||||||
|
error: wire.error,
|
||||||
|
new_segment_id: wire.new_segment_id,
|
||||||
|
};
|
||||||
|
match lifecycle.state {
|
||||||
|
CompactionLifecycleState::Running => {
|
||||||
|
lifecycle.schema_version = 3;
|
||||||
|
lifecycle.revision = lifecycle.revision.saturating_add(1);
|
||||||
|
lifecycle.state = CompactionLifecycleState::Interrupted;
|
||||||
|
lifecycle.ended_at_ms = Some(segment_log::now_millis());
|
||||||
|
lifecycle.error =
|
||||||
|
Some("worker execution restarted before compaction completed".into());
|
||||||
|
self.persist_compaction_lifecycle(&lifecycle)?;
|
||||||
|
self.send_event(Event::CompactFailed {
|
||||||
|
lifecycle: lifecycle.clone(),
|
||||||
|
});
|
||||||
|
self.release_compaction_service(&lifecycle).await;
|
||||||
|
}
|
||||||
|
CompactionLifecycleState::Interrupted => {
|
||||||
|
self.release_compaction_service(&lifecycle).await;
|
||||||
|
}
|
||||||
|
CompactionLifecycleState::Done | CompactionLifecycleState::Failed => {}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn manual_compact(&mut self) -> Result<ManualCompactResult, WorkerError> {
|
pub async fn manual_compact(&mut self) -> Result<ManualCompactResult, WorkerError> {
|
||||||
|
self.manual_compact_inner(None).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn manual_compact_with_cancel(
|
||||||
|
&mut self,
|
||||||
|
cancel: tokio::sync::watch::Receiver<bool>,
|
||||||
|
) -> Result<ManualCompactResult, WorkerError> {
|
||||||
|
self.manual_compact_inner(Some(cancel)).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn manual_compact_inner(
|
||||||
|
&mut self,
|
||||||
|
mut cancel: Option<tokio::sync::watch::Receiver<bool>>,
|
||||||
|
) -> Result<ManualCompactResult, WorkerError> {
|
||||||
if self.manifest.compaction.is_none() {
|
if self.manifest.compaction.is_none() {
|
||||||
let message =
|
let message =
|
||||||
"manual compact is unavailable because [compaction] is not configured".to_string();
|
"manual compact is unavailable because [compaction] is not configured".to_string();
|
||||||
@@ -4764,7 +4845,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
return Ok(ManualCompactResult::Skipped { message });
|
return Ok(ManualCompactResult::Skipped { message });
|
||||||
}
|
}
|
||||||
|
|
||||||
match self.compact(retained).await {
|
match self.compact_with_cancel(retained, cancel.take()).await {
|
||||||
Ok(new_segment_id) => {
|
Ok(new_segment_id) => {
|
||||||
info!(new_segment_id = %new_segment_id, "Manual compaction succeeded");
|
info!(new_segment_id = %new_segment_id, "Manual compaction succeeded");
|
||||||
if let Some(ref state) = state {
|
if let Some(ref state) = state {
|
||||||
@@ -4937,11 +5018,19 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
/// Runs one parent-owned observable compaction service and returns the new
|
/// Runs one parent-owned observable compaction service and returns the new
|
||||||
/// Segment ID. Lifecycle revisions are committed before they are broadcast.
|
/// Segment ID. Lifecycle revisions are committed before they are broadcast.
|
||||||
pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> {
|
pub async fn compact(&mut self, retained_tokens: u64) -> Result<SegmentId, WorkerError> {
|
||||||
|
self.compact_with_cancel(retained_tokens, None).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn compact_with_cancel(
|
||||||
|
&mut self,
|
||||||
|
retained_tokens: u64,
|
||||||
|
mut cancel: Option<tokio::sync::watch::Receiver<bool>>,
|
||||||
|
) -> Result<SegmentId, WorkerError> {
|
||||||
let _rewrite_guard = self
|
let _rewrite_guard = self
|
||||||
.prepare_session_rewrite(SessionRewriteKind::Compact)
|
.prepare_session_rewrite(SessionRewriteKind::Compact)
|
||||||
.await?;
|
.await?;
|
||||||
let mut lifecycle = CompactionLifecycle {
|
let mut lifecycle = CompactionLifecycle {
|
||||||
schema_version: 2,
|
schema_version: 3,
|
||||||
compaction_id: uuid::Uuid::now_v7().to_string(),
|
compaction_id: uuid::Uuid::now_v7().to_string(),
|
||||||
revision: 1,
|
revision: 1,
|
||||||
internal_worker: None,
|
internal_worker: None,
|
||||||
@@ -4953,16 +5042,25 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
new_segment_id: None,
|
new_segment_id: None,
|
||||||
};
|
};
|
||||||
self.persist_and_send_compact_start(lifecycle.clone())?;
|
self.persist_and_send_compact_start(lifecycle.clone())?;
|
||||||
match self.compact_impl(retained_tokens, &mut lifecycle).await {
|
let outcome = if let Some(cancel) = cancel.as_mut() {
|
||||||
Ok((new_segment_id, summary)) => {
|
tokio::select! {
|
||||||
lifecycle.revision = lifecycle.revision.saturating_add(1);
|
biased;
|
||||||
lifecycle.state = CompactionLifecycleState::Done;
|
changed = cancel.changed() => {
|
||||||
lifecycle.ended_at_ms = Some(segment_log::now_millis());
|
let _ = changed;
|
||||||
lifecycle.summary = Some(summary);
|
Err(WorkerError::CompactCancelled)
|
||||||
lifecycle.new_segment_id = Some(new_segment_id.to_string());
|
}
|
||||||
let terminal = self.persist_and_send_compact_done(lifecycle.clone());
|
result = self.compact_impl(retained_tokens, &mut lifecycle) => result,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.compact_impl(retained_tokens, &mut lifecycle).await
|
||||||
|
};
|
||||||
|
match outcome {
|
||||||
|
Ok((new_segment_id, _summary)) => {
|
||||||
|
debug_assert_eq!(lifecycle.state, CompactionLifecycleState::Done);
|
||||||
|
self.send_event(Event::CompactDone {
|
||||||
|
lifecycle: lifecycle.clone(),
|
||||||
|
});
|
||||||
self.release_compaction_service(&lifecycle).await;
|
self.release_compaction_service(&lifecycle).await;
|
||||||
terminal?;
|
|
||||||
Ok(new_segment_id)
|
Ok(new_segment_id)
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
@@ -5543,6 +5641,24 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
})?,
|
})?,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
// Commit the terminal lifecycle in the same atomic replacement-segment
|
||||||
|
// creation as the rewritten history. Restore can therefore never see a
|
||||||
|
// replacement segment without the Done fact for the compaction that
|
||||||
|
// created it.
|
||||||
|
lifecycle.revision = lifecycle.revision.saturating_add(1);
|
||||||
|
lifecycle.state = CompactionLifecycleState::Done;
|
||||||
|
lifecycle.ended_at_ms = Some(segment_log::now_millis());
|
||||||
|
lifecycle.summary = Some(summary_text.clone());
|
||||||
|
lifecycle.new_segment_id = Some(new_segment_id.to_string());
|
||||||
|
initial_entries.push(LogEntry::Extension {
|
||||||
|
ts: segment_log::now_millis(),
|
||||||
|
domain: COMPACTION_EXTENSION_DOMAIN.to_string(),
|
||||||
|
payload: serde_json::to_value(&*lifecycle).map_err(|error| {
|
||||||
|
WorkerError::InvalidState(format!(
|
||||||
|
"serialize terminal compaction lifecycle: {error}"
|
||||||
|
))
|
||||||
|
})?,
|
||||||
|
});
|
||||||
self.store
|
self.store
|
||||||
.create_segment(old_loc.session_id, new_segment_id, &initial_entries)?;
|
.create_segment(old_loc.session_id, new_segment_id, &initial_entries)?;
|
||||||
self.segment_state.set_location(SegmentLocation {
|
self.segment_state.set_location(SegmentLocation {
|
||||||
@@ -10165,6 +10281,56 @@ mod build_summary_prompt_tests {
|
|||||||
assert_eq!(state.notification_receipts.len(), 1);
|
assert_eq!(state.notification_receipts.len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_terminalizes_running_compaction_before_idle_publication() {
|
||||||
|
let (_dir, mut worker) = rewind_test_worker().await;
|
||||||
|
let lifecycle = CompactionLifecycle {
|
||||||
|
schema_version: 3,
|
||||||
|
compaction_id: "compact-before-restart".into(),
|
||||||
|
revision: 1,
|
||||||
|
internal_worker: None,
|
||||||
|
state: CompactionLifecycleState::Running,
|
||||||
|
started_at_ms: segment_log::now_millis(),
|
||||||
|
ended_at_ms: None,
|
||||||
|
summary: None,
|
||||||
|
error: None,
|
||||||
|
new_segment_id: None,
|
||||||
|
};
|
||||||
|
worker.persist_compaction_lifecycle(&lifecycle).unwrap();
|
||||||
|
|
||||||
|
worker.recover_unfinished_compaction().await.unwrap();
|
||||||
|
|
||||||
|
let (entries, _) = worker.sink.subscribe_with_snapshot();
|
||||||
|
let restored = entries.iter().rev().find_map(|entry| match entry {
|
||||||
|
LogEntry::Extension {
|
||||||
|
domain, payload, ..
|
||||||
|
} if domain == COMPACTION_EXTENSION_DOMAIN => {
|
||||||
|
serde_json::from_value::<CompactionLifecycle>(payload.clone()).ok()
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
});
|
||||||
|
let restored = restored.expect("terminal compaction lifecycle");
|
||||||
|
assert_eq!(restored.state, CompactionLifecycleState::Interrupted);
|
||||||
|
assert_eq!(restored.revision, 2);
|
||||||
|
assert!(
|
||||||
|
restored
|
||||||
|
.error
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|error| error.contains("restarted"))
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut future = lifecycle;
|
||||||
|
future.schema_version = 4;
|
||||||
|
future.compaction_id = "future-compaction".into();
|
||||||
|
worker.persist_compaction_lifecycle(&future).unwrap();
|
||||||
|
let error = worker.recover_unfinished_compaction().await.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error
|
||||||
|
.to_string()
|
||||||
|
.contains("unsupported compaction lifecycle schema version 4")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn minimal_manifest() -> WorkerManifest {
|
fn minimal_manifest() -> WorkerManifest {
|
||||||
let toml_str = r#"
|
let toml_str = r#"
|
||||||
[worker]
|
[worker]
|
||||||
|
|||||||
@@ -72,6 +72,41 @@ impl LlmClient for MockClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct BlockingCompactClient {
|
||||||
|
calls: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BlockingCompactClient {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
calls: Arc::new(AtomicUsize::new(0)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl LlmClient for BlockingCompactClient {
|
||||||
|
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stream(
|
||||||
|
&self,
|
||||||
|
_request: Request,
|
||||||
|
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||||
|
{
|
||||||
|
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if call == 0 {
|
||||||
|
Ok(Box::pin(futures::stream::iter(
|
||||||
|
single_text_events("seed").into_iter().map(Ok),
|
||||||
|
)))
|
||||||
|
} else {
|
||||||
|
Ok(Box::pin(futures::stream::pending()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn single_text_events(text: &str) -> Vec<LlmEvent> {
|
fn single_text_events(text: &str) -> Vec<LlmEvent> {
|
||||||
vec![
|
vec![
|
||||||
LlmEvent::text_block_start(0),
|
LlmEvent::text_block_start(0),
|
||||||
@@ -156,10 +191,10 @@ target = "./"
|
|||||||
permission = "write"
|
permission = "write"
|
||||||
"#;
|
"#;
|
||||||
|
|
||||||
async fn make_worker_with_manifest(
|
async fn make_worker_with_manifest<C>(manifest_toml: &str, client: C) -> Worker<C, TestStore>
|
||||||
manifest_toml: &str,
|
where
|
||||||
client: MockClient,
|
C: LlmClient + Clone + Send + Sync + 'static,
|
||||||
) -> Worker<MockClient, TestStore> {
|
{
|
||||||
let manifest = worker::WorkerManifest::from_toml(manifest_toml).unwrap();
|
let manifest = worker::WorkerManifest::from_toml(manifest_toml).unwrap();
|
||||||
|
|
||||||
let store_tmp = tempfile::tempdir().unwrap();
|
let store_tmp = tempfile::tempdir().unwrap();
|
||||||
@@ -614,12 +649,144 @@ async fn pre_run_compact_failure_broadcasts_start_and_failed() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn manual_compact_cancel_terminalizes_before_returning_idle() {
|
||||||
|
let worker =
|
||||||
|
make_worker_with_manifest(POST_RUN_MANIFEST_TOML, BlockingCompactClient::new()).await;
|
||||||
|
let runtime_tmp = tempfile::tempdir().unwrap();
|
||||||
|
let bash_output_dir = runtime_tmp.path().join("bash-output");
|
||||||
|
let (handle, shutdown_receiver) =
|
||||||
|
WorkerController::spawn(worker, runtime_tmp.path(), &bash_output_dir)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut rx = handle.subscribe();
|
||||||
|
|
||||||
|
handle
|
||||||
|
.send(Method::submit_text(
|
||||||
|
protocol::new_submission_request_id(),
|
||||||
|
"seed history",
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("send seed run");
|
||||||
|
loop {
|
||||||
|
if matches!(
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("timeout waiting for seed run")
|
||||||
|
.expect("event"),
|
||||||
|
Event::RunEnd {
|
||||||
|
result: RunResult::Finished
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let compact = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot());
|
||||||
|
handle
|
||||||
|
.send(Method::Compact { command: compact })
|
||||||
|
.await
|
||||||
|
.expect("send compact");
|
||||||
|
loop {
|
||||||
|
if matches!(
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("timeout waiting for compact start")
|
||||||
|
.expect("event"),
|
||||||
|
Event::CompactStart { .. }
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let cancel = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot());
|
||||||
|
handle
|
||||||
|
.send(Method::Cancel { command: cancel })
|
||||||
|
.await
|
||||||
|
.expect("send compact cancel");
|
||||||
|
let mut saw_interrupted = false;
|
||||||
|
let mut saw_idle = false;
|
||||||
|
while !(saw_interrupted && saw_idle) {
|
||||||
|
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("timeout waiting for compact cancellation")
|
||||||
|
.expect("event")
|
||||||
|
{
|
||||||
|
Event::CompactFailed { lifecycle }
|
||||||
|
if lifecycle.state == protocol::CompactionLifecycleState::Interrupted =>
|
||||||
|
{
|
||||||
|
saw_interrupted = true;
|
||||||
|
}
|
||||||
|
Event::WorkerState { snapshot }
|
||||||
|
if snapshot.catalog_status() == protocol::WorkerStatus::Idle =>
|
||||||
|
{
|
||||||
|
assert!(
|
||||||
|
saw_interrupted,
|
||||||
|
"Idle must follow durable Interrupted evidence"
|
||||||
|
);
|
||||||
|
saw_idle = true;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let compact = protocol::WorkerCommandEnvelope::for_snapshot(3, &handle.shared_state.snapshot());
|
||||||
|
handle
|
||||||
|
.send(Method::Compact { command: compact })
|
||||||
|
.await
|
||||||
|
.expect("send second compact");
|
||||||
|
loop {
|
||||||
|
if matches!(
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("timeout waiting for second compact start")
|
||||||
|
.expect("event"),
|
||||||
|
Event::CompactStart { .. }
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let shutdown =
|
||||||
|
protocol::WorkerCommandEnvelope::for_snapshot(4, &handle.shared_state.snapshot());
|
||||||
|
handle
|
||||||
|
.send(Method::Shutdown { command: shutdown })
|
||||||
|
.await
|
||||||
|
.expect("send shutdown during compact");
|
||||||
|
let mut interrupted_before_shutdown = false;
|
||||||
|
loop {
|
||||||
|
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("timeout waiting for shutdown")
|
||||||
|
.expect("event")
|
||||||
|
{
|
||||||
|
Event::CompactFailed { lifecycle }
|
||||||
|
if lifecycle.state == protocol::CompactionLifecycleState::Interrupted =>
|
||||||
|
{
|
||||||
|
interrupted_before_shutdown = true;
|
||||||
|
}
|
||||||
|
Event::Shutdown => {
|
||||||
|
assert!(
|
||||||
|
interrupted_before_shutdown,
|
||||||
|
"shutdown must await terminal compaction evidence"
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), shutdown_receiver)
|
||||||
|
.await
|
||||||
|
.expect("controller shutdown timeout")
|
||||||
|
.expect("shutdown confirmation");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn controller_compact_method_emits_start_and_done() {
|
async fn controller_compact_method_emits_start_and_done() {
|
||||||
let client = MockClient::new(vec![
|
let client = MockClient::new(vec![
|
||||||
text_events_with_usage("hi", 1000),
|
text_events_with_usage("hi", 1000),
|
||||||
write_summary_tool_use_events("manual-summary", "manual compact summary"),
|
write_summary_tool_use_events("manual-summary", "manual compact summary"),
|
||||||
single_text_events("done"),
|
single_text_events("done"),
|
||||||
|
single_text_events("follow-up"),
|
||||||
]);
|
]);
|
||||||
let worker = make_worker_with_manifest(POST_RUN_MANIFEST_TOML, client).await;
|
let worker = make_worker_with_manifest(POST_RUN_MANIFEST_TOML, client).await;
|
||||||
let runtime_tmp = tempfile::tempdir().unwrap();
|
let runtime_tmp = tempfile::tempdir().unwrap();
|
||||||
@@ -649,7 +816,11 @@ async fn controller_compact_method_emits_start_and_done() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
handle.send(Method::Compact).await.expect("send compact");
|
let command = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot());
|
||||||
|
handle
|
||||||
|
.send(Method::Compact { command })
|
||||||
|
.await
|
||||||
|
.expect("send compact");
|
||||||
let mut saw_start = false;
|
let mut saw_start = false;
|
||||||
loop {
|
loop {
|
||||||
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
@@ -670,5 +841,30 @@ async fn controller_compact_method_emits_start_and_done() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert!(saw_start, "manual compact should emit CompactStart");
|
assert!(saw_start, "manual compact should emit CompactStart");
|
||||||
let _ = handle.send(Method::Shutdown).await;
|
handle
|
||||||
|
.send(Method::submit_text(
|
||||||
|
protocol::new_submission_request_id(),
|
||||||
|
"run after compact",
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("send follow-up run");
|
||||||
|
loop {
|
||||||
|
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
|
.await
|
||||||
|
.expect("timeout waiting for follow-up run")
|
||||||
|
.expect("event")
|
||||||
|
{
|
||||||
|
Event::RunEnd {
|
||||||
|
result: RunResult::Finished,
|
||||||
|
} => break,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
handle.shared_state.catalog_status(),
|
||||||
|
protocol::WorkerStatus::Idle,
|
||||||
|
"successful manual compaction must release the execution fence"
|
||||||
|
);
|
||||||
|
let command = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot());
|
||||||
|
let _ = handle.send(Method::Shutdown { command }).await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use agen::Engine;
|
use agen::Engine;
|
||||||
@@ -25,6 +25,15 @@ use worker::{
|
|||||||
|
|
||||||
type TestStore = CombinedStore<FsStore, FsWorkerStore>;
|
type TestStore = CombinedStore<FsStore, FsWorkerStore>;
|
||||||
|
|
||||||
|
static NEXT_COMMAND_ID: AtomicU64 = AtomicU64::new(1);
|
||||||
|
|
||||||
|
fn worker_command(handle: &WorkerHandle) -> protocol::WorkerCommandEnvelope {
|
||||||
|
protocol::WorkerCommandEnvelope::for_snapshot(
|
||||||
|
NEXT_COMMAND_ID.fetch_add(1, Ordering::Relaxed),
|
||||||
|
&handle.shared_state.snapshot(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Reconstruct a worker-history-like `Vec<Item>` from the live session
|
/// Reconstruct a worker-history-like `Vec<Item>` from the live session
|
||||||
/// log mirror held by the Worker's broadcast sink. Replaces the previous
|
/// log mirror held by the Worker's broadcast sink. Replaces the previous
|
||||||
/// `WorkerSharedState.history()` test helper now that the mirror lives in
|
/// `WorkerSharedState.history()` test helper now that the mirror lives in
|
||||||
@@ -313,7 +322,12 @@ async fn controller_grants_read_scope_for_exact_bash_output_directory() {
|
|||||||
}));
|
}));
|
||||||
assert!(!handle.runtime_dir.path().join("bash-output").exists());
|
assert!(!handle.runtime_dir.path().join("bash-output").exists());
|
||||||
|
|
||||||
handle.send(Method::Shutdown).await.unwrap();
|
handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
shutdown_rx.await.unwrap();
|
shutdown_rx.await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -346,7 +360,12 @@ async fn shutdown_closes_bound_workdir_session() {
|
|||||||
WorkerController::spawn(worker, runtime_base.path(), &bash_output_dir)
|
WorkerController::spawn(worker, runtime_base.path(), &bash_output_dir)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
handle.send(Method::Shutdown).await.unwrap();
|
handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
|
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
|
||||||
.await
|
.await
|
||||||
.expect("controller should shut down")
|
.expect("controller should shut down")
|
||||||
@@ -461,7 +480,12 @@ async fn controller_projects_workdir_command_events_and_snapshot_state() {
|
|||||||
!durable_history.contains("ready") && !durable_history.contains("done"),
|
!durable_history.contains("ready") && !durable_history.contains("done"),
|
||||||
"operational command chunks must not be appended to Worker history: {durable_history}"
|
"operational command chunks must not be appended to Worker history: {durable_history}"
|
||||||
);
|
);
|
||||||
handle.send(Method::Shutdown).await.unwrap();
|
handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -533,7 +557,12 @@ async fn controller_refreshes_command_snapshot_after_high_output_provider_lag()
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(output.status, workdir::CommandStatus::Cancelled);
|
assert_eq!(output.status, workdir::CommandStatus::Cancelled);
|
||||||
handle.send(Method::Shutdown).await.unwrap();
|
handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -575,13 +604,13 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
|||||||
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
||||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||||
loop {
|
loop {
|
||||||
if handle.shared_state.get_status() == status {
|
if handle.shared_state.catalog_status() == status {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
tokio::time::Instant::now() < deadline,
|
tokio::time::Instant::now() < deadline,
|
||||||
"timed out waiting for status {status:?}; current={:?}",
|
"timed out waiting for status {status:?}; current={:?}",
|
||||||
handle.shared_state.get_status()
|
handle.shared_state.catalog_status()
|
||||||
);
|
);
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||||
}
|
}
|
||||||
@@ -1033,7 +1062,8 @@ async fn run_end_returns_to_idle_without_busy_status() {
|
|||||||
Ok(Event::RunEnd { result: protocol::RunResult::Finished }) => {
|
Ok(Event::RunEnd { result: protocol::RunResult::Finished }) => {
|
||||||
saw_run_end = true;
|
saw_run_end = true;
|
||||||
}
|
}
|
||||||
Ok(Event::Status { status: WorkerStatus::Idle }) if saw_run_end => {
|
Ok(Event::WorkerState { snapshot })
|
||||||
|
if saw_run_end && snapshot.catalog_status() == WorkerStatus::Idle => {
|
||||||
saw_idle_status = true;
|
saw_idle_status = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1050,7 +1080,7 @@ async fn run_end_returns_to_idle_without_busy_status() {
|
|||||||
saw_idle_status,
|
saw_idle_status,
|
||||||
"expected idle status immediately after RunEnd"
|
"expected idle status immediately after RunEnd"
|
||||||
);
|
);
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1128,9 +1158,7 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
|
|||||||
loop {
|
loop {
|
||||||
if matches!(
|
if matches!(
|
||||||
events.recv().await,
|
events.recv().await,
|
||||||
Ok(Event::Status {
|
Ok(Event::WorkerState { snapshot }) if snapshot.catalog_status() == WorkerStatus::Running
|
||||||
status: WorkerStatus::Running,
|
|
||||||
})
|
|
||||||
) {
|
) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1205,8 +1233,8 @@ async fn attach_snapshot_includes_current_status() {
|
|||||||
loop {
|
loop {
|
||||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||||
match event {
|
match event {
|
||||||
Event::Snapshot { status, .. } => {
|
Event::Snapshot { state, .. } => {
|
||||||
assert_eq!(status, WorkerStatus::Running);
|
assert_eq!(state.catalog_status(), WorkerStatus::Running);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Event::Alert(_) => continue,
|
Event::Alert(_) => continue,
|
||||||
@@ -1221,7 +1249,7 @@ async fn shared_state_starts_idle() {
|
|||||||
let worker = make_worker(client).await;
|
let worker = make_worker(client).await;
|
||||||
let handle = spawn_controller(worker).await;
|
let handle = spawn_controller(worker).await;
|
||||||
|
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1241,7 +1269,7 @@ async fn run_updates_shared_state_to_idle_after_completion() {
|
|||||||
// Wait for the run to complete
|
// Wait for the run to complete
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1364,7 +1392,12 @@ async fn submit_while_running_is_durably_queued() {
|
|||||||
assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued));
|
assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued));
|
||||||
let pending_snapshot = pending_snapshot.expect("pending snapshot");
|
let pending_snapshot = pending_snapshot.expect("pending snapshot");
|
||||||
assert_eq!(pending_snapshot.submissions.len(), 1);
|
assert_eq!(pending_snapshot.submissions.len(), 1);
|
||||||
handle.send(Method::Pause).await.unwrap();
|
handle
|
||||||
|
.send(Method::Pause {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
wait_for_status(&handle, WorkerStatus::Paused).await;
|
wait_for_status(&handle, WorkerStatus::Paused).await;
|
||||||
handle
|
handle
|
||||||
.send(Method::ContinuePending {
|
.send(Method::ContinuePending {
|
||||||
@@ -1386,17 +1419,22 @@ async fn submit_while_running_is_durably_queued() {
|
|||||||
.await
|
.await
|
||||||
.expect("paused ContinuePending rejection");
|
.expect("paused ContinuePending rejection");
|
||||||
assert!(rejection.contains("Resume or Cancel"));
|
assert!(rejection.contains("Resume or Cancel"));
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn resume_without_pause_returns_error() {
|
async fn resume_without_pause_returns_invalid_state_acknowledgement() {
|
||||||
let client = MockClient::new(simple_text_events());
|
let client = MockClient::new(simple_text_events());
|
||||||
let worker = make_worker(client).await;
|
let worker = make_worker(client).await;
|
||||||
let handle = spawn_controller(worker).await;
|
let handle = spawn_controller(worker).await;
|
||||||
let mut rx = handle.subscribe();
|
let mut rx = handle.subscribe();
|
||||||
|
|
||||||
handle.send(Method::Resume).await.unwrap();
|
handle
|
||||||
|
.send(Method::Resume {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let mut saw_not_paused = false;
|
let mut saw_not_paused = false;
|
||||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||||
@@ -1404,7 +1442,10 @@ async fn resume_without_pause_returns_error() {
|
|||||||
tokio::select! {
|
tokio::select! {
|
||||||
event = rx.recv() => {
|
event = rx.recv() => {
|
||||||
match event {
|
match event {
|
||||||
Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotPaused => {
|
Ok(Event::CommandAcknowledged { acknowledgement })
|
||||||
|
if acknowledgement.command == protocol::WorkerCommandKind::Resume
|
||||||
|
&& acknowledgement.disposition
|
||||||
|
== protocol::WorkerCommandDisposition::InvalidState => {
|
||||||
saw_not_paused = true;
|
saw_not_paused = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1416,17 +1457,22 @@ async fn resume_without_pause_returns_error() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
assert!(saw_not_paused, "should see not_paused error");
|
assert!(saw_not_paused, "should see invalid-state acknowledgement");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn cancel_without_run_returns_error() {
|
async fn cancel_without_run_returns_invalid_state_acknowledgement() {
|
||||||
let client = MockClient::new(simple_text_events());
|
let client = MockClient::new(simple_text_events());
|
||||||
let worker = make_worker(client).await;
|
let worker = make_worker(client).await;
|
||||||
let handle = spawn_controller(worker).await;
|
let handle = spawn_controller(worker).await;
|
||||||
let mut rx = handle.subscribe();
|
let mut rx = handle.subscribe();
|
||||||
|
|
||||||
handle.send(Method::Cancel).await.unwrap();
|
handle
|
||||||
|
.send(Method::Cancel {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let mut saw_not_running = false;
|
let mut saw_not_running = false;
|
||||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||||
@@ -1434,7 +1480,10 @@ async fn cancel_without_run_returns_error() {
|
|||||||
tokio::select! {
|
tokio::select! {
|
||||||
event = rx.recv() => {
|
event = rx.recv() => {
|
||||||
match event {
|
match event {
|
||||||
Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotRunning => {
|
Ok(Event::CommandAcknowledged { acknowledgement })
|
||||||
|
if acknowledgement.command == protocol::WorkerCommandKind::Cancel
|
||||||
|
&& acknowledgement.disposition
|
||||||
|
== protocol::WorkerCommandDisposition::InvalidState => {
|
||||||
saw_not_running = true;
|
saw_not_running = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1446,7 +1495,7 @@ async fn cancel_without_run_returns_error() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
assert!(saw_not_running, "should see not_running error");
|
assert!(saw_not_running, "should see invalid-state acknowledgement");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1822,7 +1871,7 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||||
assert!(
|
assert!(
|
||||||
client_for_assert.captured_requests().is_empty(),
|
client_for_assert.captured_requests().is_empty(),
|
||||||
"weak Notify must not stage RunForNotification while idle"
|
"weak Notify must not stage RunForNotification while idle"
|
||||||
@@ -1919,7 +1968,7 @@ async fn worker_event_turn_ended_while_idle_auto_starts_turn_and_injects_system_
|
|||||||
saw_worker_event_in_mirror,
|
saw_worker_event_in_mirror,
|
||||||
"Method::WorkerEvent should commit a SystemItem::WorkerEvent entry"
|
"Method::WorkerEvent should commit a SystemItem::WorkerEvent entry"
|
||||||
);
|
);
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||||
|
|
||||||
let requests = client_for_assert.captured_requests();
|
let requests = client_for_assert.captured_requests();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -1982,7 +2031,7 @@ async fn worker_event_scope_sub_delegated_while_idle_stays_control_plane_only()
|
|||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
handle.shared_state.get_status(),
|
handle.shared_state.catalog_status(),
|
||||||
WorkerStatus::Idle,
|
WorkerStatus::Idle,
|
||||||
"control-plane ScopeSubDelegated must not auto-start the parent LLM"
|
"control-plane ScopeSubDelegated must not auto-start the parent LLM"
|
||||||
);
|
);
|
||||||
@@ -2085,7 +2134,12 @@ async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
handle.send(Method::Cancel).await.unwrap();
|
handle
|
||||||
|
.send(Method::Cancel {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||||
|
|
||||||
let mut rx = handle.subscribe();
|
let mut rx = handle.subscribe();
|
||||||
@@ -2482,7 +2536,12 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
|||||||
"text_delta should arrive before pause"
|
"text_delta should arrive before pause"
|
||||||
);
|
);
|
||||||
|
|
||||||
handle.send(Method::Pause).await.unwrap();
|
handle
|
||||||
|
.send(Method::Pause {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// The controller emits RunEnd { Paused } when the
|
// The controller emits RunEnd { Paused } when the
|
||||||
// EngineError::Cancelled is translated under pause_requested.
|
// EngineError::Cancelled is translated under pause_requested.
|
||||||
@@ -2498,9 +2557,14 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
|
||||||
|
|
||||||
handle.send(Method::Resume).await.unwrap();
|
handle
|
||||||
|
.send(Method::Resume {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
@@ -2514,7 +2578,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||||
|
|
||||||
// History consistency: exactly [user "hello", assistant
|
// History consistency: exactly [user "hello", assistant
|
||||||
// "resumed output"]. No artifacts from the aborted stream
|
// "resumed output"]. No artifacts from the aborted stream
|
||||||
@@ -2614,7 +2678,12 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
|
|||||||
"tool_call_done should arrive before pause"
|
"tool_call_done should arrive before pause"
|
||||||
);
|
);
|
||||||
|
|
||||||
handle.send(Method::Pause).await.unwrap();
|
handle
|
||||||
|
.send(Method::Pause {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
e,
|
e,
|
||||||
@@ -2626,7 +2695,7 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() {
|
|||||||
"expected RunEnd::Paused"
|
"expected RunEnd::Paused"
|
||||||
);
|
);
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused);
|
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
|
||||||
|
|
||||||
// New user input while Paused → `Worker::run` observes
|
// New user input while Paused → `Worker::run` observes
|
||||||
// `last_run_interrupted` and runs its interrupt-prep step, which
|
// `last_run_interrupted` and runs its interrupt-prep step, which
|
||||||
@@ -2785,7 +2854,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
|||||||
"tool_call_done should arrive before pause"
|
"tool_call_done should arrive before pause"
|
||||||
);
|
);
|
||||||
|
|
||||||
handle.send(Method::Pause).await.unwrap();
|
handle
|
||||||
|
.send(Method::Pause {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
e,
|
e,
|
||||||
@@ -2798,7 +2872,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
|||||||
);
|
);
|
||||||
wait_for_status(&handle, WorkerStatus::Paused).await;
|
wait_for_status(&handle, WorkerStatus::Paused).await;
|
||||||
|
|
||||||
handle.send(Method::Cancel).await.unwrap();
|
handle
|
||||||
|
.send(Method::Cancel {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||||
let (entries_after_cancel, _rx_after_cancel) = handle.sink.subscribe_with_snapshot();
|
let (entries_after_cancel, _rx_after_cancel) = handle.sink.subscribe_with_snapshot();
|
||||||
assert!(
|
assert!(
|
||||||
@@ -2824,17 +2903,22 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() {
|
|||||||
"paused cancel must not resume or start another LLM request"
|
"paused cancel must not resume or start another LLM request"
|
||||||
);
|
);
|
||||||
|
|
||||||
handle.send(Method::Resume).await.unwrap();
|
handle
|
||||||
|
.send(Method::Resume {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
e,
|
e,
|
||||||
Event::Error {
|
Event::CommandAcknowledged { acknowledgement }
|
||||||
code: worker::ErrorCode::NotPaused,
|
if acknowledgement.command == protocol::WorkerCommandKind::Resume
|
||||||
..
|
&& acknowledgement.disposition
|
||||||
}
|
== protocol::WorkerCommandDisposition::InvalidState
|
||||||
))
|
))
|
||||||
.await,
|
.await,
|
||||||
"resume after paused cancel should be rejected as not paused"
|
"resume after paused cancel should receive invalid-state acknowledgement"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
client_for_assert.captured_requests().len(),
|
client_for_assert.captured_requests().len(),
|
||||||
@@ -2943,7 +3027,12 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||||
handle.send(Method::Cancel).await.unwrap();
|
handle
|
||||||
|
.send(Method::Cancel {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
@@ -2981,7 +3070,12 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||||
handle.send(Method::Pause).await.unwrap();
|
handle
|
||||||
|
.send(Method::Pause {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
@@ -3038,7 +3132,12 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||||
handle.send(Method::Cancel).await.unwrap();
|
handle
|
||||||
|
.send(Method::Cancel {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
e,
|
e,
|
||||||
@@ -3095,7 +3194,12 @@ async fn pause_after_assistant_token_does_not_rollback() {
|
|||||||
.await,
|
.await,
|
||||||
"assistant token should be visible before pause"
|
"assistant token should be visible before pause"
|
||||||
);
|
);
|
||||||
handle.send(Method::Pause).await.unwrap();
|
handle
|
||||||
|
.send(Method::Pause {
|
||||||
|
command: worker_command(&handle),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!(
|
||||||
|
|||||||
@@ -1904,7 +1904,12 @@ pub struct WorkerSummary {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub tags: Vec<String>,
|
pub tags: Vec<String>,
|
||||||
pub workspace: WorkerWorkspaceSummary,
|
pub workspace: WorkerWorkspaceSummary,
|
||||||
|
/// Runtime catalog lifecycle compatibility state. Live foreground state, when
|
||||||
|
/// available, is carried separately in `worker_state`.
|
||||||
pub state: String,
|
pub state: String,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[cfg_attr(feature = "typescript", ts(optional))]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
pub last_seen_at: Option<String>,
|
pub last_seen_at: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub pinned: bool,
|
pub pinned: bool,
|
||||||
|
|||||||
@@ -36,8 +36,6 @@ use worker_runtime::config_bundle::{
|
|||||||
ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
|
ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor,
|
||||||
};
|
};
|
||||||
use worker_runtime::error::RuntimeError as EmbeddedRuntimeError;
|
use worker_runtime::error::RuntimeError as EmbeddedRuntimeError;
|
||||||
#[cfg(test)]
|
|
||||||
use worker_runtime::execution::WorkerExecutionRunState;
|
|
||||||
use worker_runtime::fs_store::FsRuntimeStoreOptions;
|
use worker_runtime::fs_store::FsRuntimeStoreOptions;
|
||||||
use worker_runtime::http_server::{
|
use worker_runtime::http_server::{
|
||||||
RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER,
|
RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER,
|
||||||
@@ -245,7 +243,10 @@ pub struct WorkerSummary {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub tags: Vec<String>,
|
pub tags: Vec<String>,
|
||||||
pub workspace: WorkerWorkspaceSummary,
|
pub workspace: WorkerWorkspaceSummary,
|
||||||
|
/// Runtime catalog lifecycle compatibility state.
|
||||||
pub state: String,
|
pub state: String,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
pub last_seen_at: Option<String>,
|
pub last_seen_at: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub pinned: bool,
|
pub pinned: bool,
|
||||||
@@ -337,6 +338,7 @@ pub(crate) fn workspace_worker_summary(
|
|||||||
workspace_id: summary.workspace.workspace_id,
|
workspace_id: summary.workspace.workspace_id,
|
||||||
},
|
},
|
||||||
state: summary.state,
|
state: summary.state,
|
||||||
|
worker_state: summary.worker_state,
|
||||||
last_seen_at: summary.last_seen_at,
|
last_seen_at: summary.last_seen_at,
|
||||||
pinned: summary.pinned,
|
pinned: summary.pinned,
|
||||||
retention_state: summary.retention_state,
|
retention_state: summary.retention_state,
|
||||||
@@ -2000,6 +2002,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
workspace_id: summary.workspace_id.clone(),
|
workspace_id: summary.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(summary.status).to_string(),
|
state: embedded_worker_status_label(summary.status).to_string(),
|
||||||
|
worker_state: summary.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -2039,6 +2042,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
workspace_id: detail.workspace_id.clone(),
|
workspace_id: detail.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(detail.status).to_string(),
|
state: embedded_worker_status_label(detail.status).to_string(),
|
||||||
|
worker_state: detail.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -3342,6 +3346,7 @@ impl RemoteWorkerRuntime {
|
|||||||
workspace_id: summary.workspace_id.clone(),
|
workspace_id: summary.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(summary.status).to_string(),
|
state: embedded_worker_status_label(summary.status).to_string(),
|
||||||
|
worker_state: summary.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -3385,6 +3390,7 @@ impl RemoteWorkerRuntime {
|
|||||||
workspace_id: detail.workspace_id.clone(),
|
workspace_id: detail.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(detail.status).to_string(),
|
state: embedded_worker_status_label(detail.status).to_string(),
|
||||||
|
worker_state: detail.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -4732,6 +4738,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> WorkerSummary {
|
|||||||
workspace_id: None,
|
workspace_id: None,
|
||||||
},
|
},
|
||||||
state: "unsupported".to_string(),
|
state: "unsupported".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -5170,7 +5177,6 @@ mod tests {
|
|||||||
request.worker_ref,
|
request.worker_ref,
|
||||||
self.backend_id(),
|
self.backend_id(),
|
||||||
),
|
),
|
||||||
run_state: WorkerExecutionRunState::Idle,
|
|
||||||
working_directory: request
|
working_directory: request
|
||||||
.working_directory
|
.working_directory
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -5199,8 +5205,8 @@ mod tests {
|
|||||||
let content = input.content;
|
let content = input.content;
|
||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||||
let _ = context.publish_protocol_event(protocol::Event::Status {
|
let _ = context.publish_protocol_event(protocol::Event::WorkerState {
|
||||||
status: protocol::WorkerStatus::Running,
|
snapshot: protocol::WorkerStatus::Running.into(),
|
||||||
});
|
});
|
||||||
let _ = context.publish_protocol_event(protocol::Event::TextDone {
|
let _ = context.publish_protocol_event(protocol::Event::TextDone {
|
||||||
text: format!("echo: {content}"),
|
text: format!("echo: {content}"),
|
||||||
@@ -5208,14 +5214,13 @@ mod tests {
|
|||||||
let _ = context.publish_protocol_event(protocol::Event::RunEnd {
|
let _ = context.publish_protocol_event(protocol::Event::RunEnd {
|
||||||
result: protocol::RunResult::Finished,
|
result: protocol::RunResult::Finished,
|
||||||
});
|
});
|
||||||
let _ = context.publish_protocol_event(protocol::Event::Status {
|
let _ = context.publish_protocol_event(protocol::Event::WorkerState {
|
||||||
status: protocol::WorkerStatus::Idle,
|
snapshot: protocol::WorkerStatus::Idle.into(),
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
if let Some(submission_request_id) = submission_request_id {
|
if let Some(submission_request_id) = submission_request_id {
|
||||||
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
||||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Busy,
|
|
||||||
submission_request_id,
|
submission_request_id,
|
||||||
uuid::Uuid::now_v7().to_string(),
|
uuid::Uuid::now_v7().to_string(),
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
@@ -5223,7 +5228,6 @@ mod tests {
|
|||||||
} else {
|
} else {
|
||||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Busy,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5256,6 +5260,7 @@ mod tests {
|
|||||||
workspace_id: None,
|
workspace_id: None,
|
||||||
},
|
},
|
||||||
state: "available".to_string(),
|
state: "available".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use worker_runtime::catalog::{
|
|||||||
};
|
};
|
||||||
use worker_runtime::execution::{
|
use worker_runtime::execution::{
|
||||||
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult,
|
WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult,
|
||||||
WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult,
|
||||||
};
|
};
|
||||||
use worker_runtime::identity::WorkerId;
|
use worker_runtime::identity::WorkerId;
|
||||||
use worker_runtime::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary};
|
use worker_runtime::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary};
|
||||||
@@ -22,7 +22,6 @@ impl WorkerExecutionBackend for TestExecutionBackend {
|
|||||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
WorkerExecutionSpawnResult::connected(
|
WorkerExecutionSpawnResult::connected(
|
||||||
WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
WorkerExecutionHandle::new(request.worker_ref, self.backend_id()),
|
||||||
WorkerExecutionRunState::Idle,
|
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -35,24 +34,17 @@ impl WorkerExecutionBackend for TestExecutionBackend {
|
|||||||
if let Some(submission_request_id) = input.submission_request_id {
|
if let Some(submission_request_id) = input.submission_request_id {
|
||||||
WorkerExecutionResult::accepted_submission(
|
WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
WorkerExecutionRunState::Busy,
|
|
||||||
submission_request_id,
|
submission_request_id,
|
||||||
uuid::Uuid::now_v7().to_string(),
|
uuid::Uuid::now_v7().to_string(),
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Input)
|
||||||
WorkerExecutionOperation::Input,
|
|
||||||
WorkerExecutionRunState::Busy,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult {
|
||||||
WorkerExecutionResult::accepted(
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)
|
||||||
WorkerExecutionOperation::Stop,
|
|
||||||
WorkerExecutionRunState::Stopped,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -199,8 +191,8 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
|||||||
runtime
|
runtime
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
&worker.worker_ref,
|
&worker.worker_ref,
|
||||||
protocol::Event::Status {
|
protocol::Event::WorkerState {
|
||||||
status: protocol::WorkerStatus::Running,
|
snapshot: protocol::WorkerStatus::Running.into(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -210,7 +202,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
|||||||
BrokerSubscriptionEvent::Event {
|
BrokerSubscriptionEvent::Event {
|
||||||
payload: SubscriptionEventPayload::WorkerUpserted { ref worker },
|
payload: SubscriptionEventPayload::WorkerUpserted { ref worker },
|
||||||
..
|
..
|
||||||
} if worker.state == SubscriptionWorkerState::Running
|
} if worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running
|
||||||
|
)),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
||||||
@@ -220,7 +221,18 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
|||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
snapshot,
|
snapshot,
|
||||||
SubscriptionSnapshot::Workers { workers }
|
SubscriptionSnapshot::Workers { workers }
|
||||||
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
if workers.iter().any(|worker| {
|
||||||
|
worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(
|
||||||
|
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
|
||||||
|
),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
|
})
|
||||||
));
|
));
|
||||||
drop(late);
|
drop(late);
|
||||||
|
|
||||||
@@ -339,14 +351,24 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
|
|||||||
runtime
|
runtime
|
||||||
.observe_worker_event(
|
.observe_worker_event(
|
||||||
&worker.worker_ref,
|
&worker.worker_ref,
|
||||||
protocol::Event::Status {
|
protocol::Event::WorkerState {
|
||||||
status: protocol::WorkerStatus::Running,
|
snapshot: protocol::WorkerStatus::Running.into(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(matches!(next_event(&mut subscription).await,
|
assert!(matches!(next_event(&mut subscription).await,
|
||||||
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
|
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
|
||||||
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running));
|
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime")
|
||||||
|
&& worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running
|
||||||
|
)),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)));
|
||||||
let mut late = broker
|
let mut late = broker
|
||||||
.subscribe(
|
.subscribe(
|
||||||
"embedded-worker-runtime",
|
"embedded-worker-runtime",
|
||||||
@@ -359,7 +381,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
|
|||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
snapshot,
|
snapshot,
|
||||||
SubscriptionSnapshot::Workers { workers }
|
SubscriptionSnapshot::Workers { workers }
|
||||||
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
if workers.iter().any(|worker| {
|
||||||
|
worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(
|
||||||
|
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
|
||||||
|
),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
|
})
|
||||||
));
|
));
|
||||||
|
|
||||||
runtime
|
runtime
|
||||||
|
|||||||
@@ -15766,6 +15766,7 @@ fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> WorkerSummary
|
|||||||
singleton_key: None,
|
singleton_key: None,
|
||||||
tags: Vec::new(),
|
tags: Vec::new(),
|
||||||
state: "missing".to_string(),
|
state: "missing".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: Some(record.updated_at.clone()),
|
last_seen_at: Some(record.updated_at.clone()),
|
||||||
pinned: record.retention_state == "pinned",
|
pinned: record.retention_state == "pinned",
|
||||||
retention_state: record.retention_state.clone(),
|
retention_state: record.retention_state.clone(),
|
||||||
@@ -19422,7 +19423,6 @@ mod tests {
|
|||||||
request.worker_ref,
|
request.worker_ref,
|
||||||
self.backend_id(),
|
self.backend_id(),
|
||||||
),
|
),
|
||||||
run_state: worker_runtime::execution::WorkerExecutionRunState::Idle,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -19438,17 +19438,43 @@ mod tests {
|
|||||||
.push((handle.worker_ref().clone(), method));
|
.push((handle.worker_ref().clone(), method));
|
||||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||||
worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod,
|
worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod,
|
||||||
worker_runtime::execution::WorkerExecutionRunState::Idle,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn worker_snapshot(
|
||||||
|
&self,
|
||||||
|
handle: &worker_runtime::execution::WorkerExecutionHandle,
|
||||||
|
) -> Option<protocol::Event> {
|
||||||
|
Some(protocol::Event::Snapshot {
|
||||||
|
session: protocol::SessionSnapshot {
|
||||||
|
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||||
|
entries: Vec::new(),
|
||||||
|
},
|
||||||
|
greeting: protocol::Greeting {
|
||||||
|
worker_name: handle.worker_ref().worker_id.to_string(),
|
||||||
|
cwd: String::new(),
|
||||||
|
provider: "deterministic-workspace-server-test".to_string(),
|
||||||
|
model: "deterministic-workspace-server-test".to_string(),
|
||||||
|
scope_summary: "test execution snapshot".to_string(),
|
||||||
|
tools: Vec::new(),
|
||||||
|
context_window: 0,
|
||||||
|
context_tokens: 0,
|
||||||
|
},
|
||||||
|
state: protocol::WorkerStateSnapshot::initial(1),
|
||||||
|
in_flight: protocol::InFlightSnapshot {
|
||||||
|
blocks: Vec::new(),
|
||||||
|
commands: Vec::new(),
|
||||||
|
},
|
||||||
|
internal_workers: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn stop_worker(
|
fn stop_worker(
|
||||||
&self,
|
&self,
|
||||||
_handle: &worker_runtime::execution::WorkerExecutionHandle,
|
_handle: &worker_runtime::execution::WorkerExecutionHandle,
|
||||||
) -> worker_runtime::execution::WorkerExecutionResult {
|
) -> worker_runtime::execution::WorkerExecutionResult {
|
||||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||||
worker_runtime::execution::WorkerExecutionOperation::Stop,
|
worker_runtime::execution::WorkerExecutionOperation::Stop,
|
||||||
worker_runtime::execution::WorkerExecutionRunState::Stopped,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -19458,7 +19484,6 @@ mod tests {
|
|||||||
) -> worker_runtime::execution::WorkerExecutionResult {
|
) -> worker_runtime::execution::WorkerExecutionResult {
|
||||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||||
worker_runtime::execution::WorkerExecutionOperation::Cancel,
|
worker_runtime::execution::WorkerExecutionOperation::Cancel,
|
||||||
worker_runtime::execution::WorkerExecutionRunState::Stopped,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -19495,16 +19520,16 @@ mod tests {
|
|||||||
if let Some(submission_request_id) = submission_request_id {
|
if let Some(submission_request_id) = submission_request_id {
|
||||||
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
worker_runtime::execution::WorkerExecutionResult::accepted_submission(
|
||||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||||
worker_runtime::execution::WorkerExecutionRunState::Idle,
|
|
||||||
submission_request_id,
|
submission_request_id,
|
||||||
uuid::Uuid::now_v7().to_string(),
|
uuid::Uuid::now_v7().to_string(),
|
||||||
protocol::SubmissionDisposition::Started,
|
protocol::SubmissionDisposition::Started,
|
||||||
)
|
)
|
||||||
|
.with_worker_state(protocol::WorkerStateSnapshot::initial(1))
|
||||||
} else {
|
} else {
|
||||||
worker_runtime::execution::WorkerExecutionResult::accepted(
|
worker_runtime::execution::WorkerExecutionResult::accepted(
|
||||||
worker_runtime::execution::WorkerExecutionOperation::Input,
|
worker_runtime::execution::WorkerExecutionOperation::Input,
|
||||||
worker_runtime::execution::WorkerExecutionRunState::Idle,
|
|
||||||
)
|
)
|
||||||
|
.with_worker_state(protocol::WorkerStateSnapshot::initial(1))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -25570,6 +25595,7 @@ mod tests {
|
|||||||
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
||||||
},
|
},
|
||||||
state: "idle".to_string(),
|
state: "idle".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "normal".to_string(),
|
retention_state: "normal".to_string(),
|
||||||
@@ -25666,6 +25692,7 @@ mod tests {
|
|||||||
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
||||||
},
|
},
|
||||||
state: "idle".to_string(),
|
state: "idle".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "normal".to_string(),
|
retention_state: "normal".to_string(),
|
||||||
@@ -28556,7 +28583,13 @@ mod tests {
|
|||||||
protocol::subscription::SubscriptionFramePayload::WorkerProtocol(
|
protocol::subscription::SubscriptionFramePayload::WorkerProtocol(
|
||||||
protocol::subscription::SubscriptionWorkerProtocolMethod {
|
protocol::subscription::SubscriptionWorkerProtocolMethod {
|
||||||
subscription_id: second_protocol_subscription_id,
|
subscription_id: second_protocol_subscription_id,
|
||||||
method: protocol::Method::Resume,
|
method: protocol::Method::Resume {
|
||||||
|
command: protocol::WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 1,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
@@ -28573,7 +28606,7 @@ mod tests {
|
|||||||
.iter()
|
.iter()
|
||||||
.any(|(worker_ref, method)| {
|
.any(|(worker_ref, method)| {
|
||||||
worker_ref.worker_id.to_string() == worker_id
|
worker_ref.worker_id.to_string() == worker_id
|
||||||
&& matches!(method, protocol::Method::Resume)
|
&& matches!(method, protocol::Method::Resume { .. })
|
||||||
})
|
})
|
||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
@@ -28586,7 +28619,7 @@ mod tests {
|
|||||||
let protocol_methods = execution_backend.protocol_methods();
|
let protocol_methods = execution_backend.protocol_methods();
|
||||||
assert!(protocol_methods.iter().any(|(worker_ref, method)| {
|
assert!(protocol_methods.iter().any(|(worker_ref, method)| {
|
||||||
worker_ref.worker_id.to_string() == worker_id
|
worker_ref.worker_id.to_string() == worker_id
|
||||||
&& matches!(method, protocol::Method::Resume)
|
&& matches!(method, protocol::Method::Resume { .. })
|
||||||
}));
|
}));
|
||||||
server.abort();
|
server.abort();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,37 @@ export type CompletionKind = "file";
|
|||||||
|
|
||||||
export type WorkerStatus = "idle" | "running" | "paused" | "stopped";
|
export type WorkerStatus = "idle" | "running" | "paused" | "stopped";
|
||||||
|
|
||||||
|
export type WorkerCommandEnvelope = {
|
||||||
|
/**
|
||||||
|
* Caller-owned sequence. A controller accepts command ids in strictly
|
||||||
|
* increasing order for one execution generation.
|
||||||
|
*/
|
||||||
|
command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, };
|
||||||
|
|
||||||
|
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
|
||||||
|
|
||||||
|
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
|
||||||
|
|
||||||
|
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
|
||||||
|
/**
|
||||||
|
* The complete authoritative state observed after command admission.
|
||||||
|
*/
|
||||||
|
state: WorkerStateSnapshot, };
|
||||||
|
|
||||||
|
export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
|
||||||
|
|
||||||
|
export type WorkerMaintenanceState = "compacting";
|
||||||
|
|
||||||
|
export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "kind": "maintenance", "state": WorkerMaintenanceState };
|
||||||
|
|
||||||
|
export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState };
|
||||||
|
|
||||||
|
export type WorkerStateSnapshot = { execution_generation: number, revision: number,
|
||||||
|
/**
|
||||||
|
* Highest lifecycle command id observed by this controller generation.
|
||||||
|
*/
|
||||||
|
last_command_id: number, state: WorkerState, };
|
||||||
|
|
||||||
export type TurnResult = "finished" | "paused";
|
export type TurnResult = "finished" | "paused";
|
||||||
|
|
||||||
export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup";
|
export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup";
|
||||||
@@ -202,7 +233,16 @@ resource_key?: string | null,
|
|||||||
/**
|
/**
|
||||||
* Producer-owned monotonic revision for this Worker subject.
|
* Producer-owned monotonic revision for this Worker subject.
|
||||||
*/
|
*/
|
||||||
subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
subject_revision: number,
|
||||||
|
/**
|
||||||
|
* Latest revisioned foreground state observed from the Worker. This remains
|
||||||
|
* absent until an authoritative Worker snapshot/event has been applied.
|
||||||
|
*/
|
||||||
|
worker_state?: WorkerStateSnapshot | null,
|
||||||
|
/**
|
||||||
|
* Runtime catalog lifecycle compatibility projection; not foreground-state authority.
|
||||||
|
*/
|
||||||
|
state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
||||||
/**
|
/**
|
||||||
* Workspace-facing Repository key. Runtime producers leave this unset and
|
* Workspace-facing Repository key. Runtime producers leave this unset and
|
||||||
* Workspace Server projections replace `repository_id` with this field.
|
* Workspace Server projections replace `repository_id` with this field.
|
||||||
@@ -231,7 +271,7 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip
|
|||||||
|
|
||||||
export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod });
|
export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod });
|
||||||
|
|
||||||
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
||||||
|
|
||||||
export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
|
export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
|
||||||
/**
|
/**
|
||||||
@@ -247,7 +287,12 @@ summary: string,
|
|||||||
* Full tool output. Absent when the tool chose to return
|
* Full tool output. Absent when the tool chose to return
|
||||||
* summary-only, or when the result was pruned.
|
* summary-only, or when the result was pruned.
|
||||||
*/
|
*/
|
||||||
output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, status: WorkerStatus,
|
output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting,
|
||||||
|
/**
|
||||||
|
* Full revisioned live execution state. `Stopped` remains Runtime
|
||||||
|
* catalog authority and is deliberately not represented here.
|
||||||
|
*/
|
||||||
|
state: WorkerStateSnapshot,
|
||||||
/**
|
/**
|
||||||
* Unfinished model output that has already streamed in the current
|
* Unfinished model output that has already streamed in the current
|
||||||
* run but is not yet represented by committed snapshot entries.
|
* run but is not yet represented by committed snapshot entries.
|
||||||
@@ -257,4 +302,4 @@ in_flight?: InFlightSnapshot,
|
|||||||
* Parent-owned Internal Worker sessions visible to this client.
|
* Parent-owned Internal Worker sessions visible to this client.
|
||||||
* Service-private Internal Workers are deliberately excluded.
|
* Service-private Internal Workers are deliberately excluded.
|
||||||
*/
|
*/
|
||||||
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" };
|
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" };
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import type { Event } from "$lib/generated/protocol";
|
import type { Event, WorkerStateSnapshot, WorkerStatus } from "$lib/generated/protocol";
|
||||||
import {
|
import {
|
||||||
type ConsoleEventInput,
|
type ConsoleEventInput,
|
||||||
type ConsoleLine,
|
type ConsoleLine,
|
||||||
@@ -19,6 +19,23 @@ declare const Deno: {
|
|||||||
test(name: string, fn: () => void): void;
|
test(name: string, fn: () => void): void;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function workerState(status: WorkerStatus): WorkerStateSnapshot {
|
||||||
|
return {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: status === "idle" ? 0 : 1,
|
||||||
|
last_command_id: 0,
|
||||||
|
state: status === "idle"
|
||||||
|
? { kind: "idle" }
|
||||||
|
: {
|
||||||
|
kind: "busy",
|
||||||
|
state: {
|
||||||
|
kind: "run",
|
||||||
|
state: status === "paused" ? "paused" : "running",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function assert(condition: unknown, message: string): asserts condition {
|
function assert(condition: unknown, message: string): asserts condition {
|
||||||
if (!condition) {
|
if (!condition) {
|
||||||
throw new Error(message);
|
throw new Error(message);
|
||||||
@@ -131,7 +148,7 @@ function snapshotEvent(cwd: string, entries: unknown[] = []): Event {
|
|||||||
context_window: 100,
|
context_window: 100,
|
||||||
context_tokens: 20,
|
context_tokens: 20,
|
||||||
},
|
},
|
||||||
status: "idle",
|
state: workerState("idle"),
|
||||||
in_flight: { blocks: [] },
|
in_flight: { blocks: [] },
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -201,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", ()
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
|
||||||
|
const projector = createConsoleProjector();
|
||||||
|
const running: WorkerStateSnapshot = {
|
||||||
|
execution_generation: 4,
|
||||||
|
revision: 3,
|
||||||
|
last_command_id: 2,
|
||||||
|
state: { kind: "busy", state: { kind: "run", state: "running" } },
|
||||||
|
};
|
||||||
|
const paused: WorkerStateSnapshot = {
|
||||||
|
...running,
|
||||||
|
revision: 4,
|
||||||
|
last_command_id: 3,
|
||||||
|
state: { kind: "busy", state: { kind: "run", state: "paused" } },
|
||||||
|
};
|
||||||
|
let projection = projector.append([
|
||||||
|
{
|
||||||
|
eventId: "running",
|
||||||
|
event: { event: "worker_state", data: { snapshot: running } },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventId: "stale",
|
||||||
|
event: {
|
||||||
|
event: "worker_state",
|
||||||
|
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventId: "pause-ack",
|
||||||
|
event: {
|
||||||
|
event: "command_acknowledged",
|
||||||
|
data: {
|
||||||
|
acknowledgement: {
|
||||||
|
command_id: 3,
|
||||||
|
command: "pause",
|
||||||
|
disposition: "accepted",
|
||||||
|
state: paused,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
assertEquals(projection.workerState, paused);
|
||||||
|
assertEquals(projection.status, "paused");
|
||||||
|
|
||||||
|
projection = projector.append([{
|
||||||
|
eventId: "conflict",
|
||||||
|
event: {
|
||||||
|
event: "worker_state",
|
||||||
|
data: { snapshot: { ...paused, state: { kind: "idle" } } },
|
||||||
|
},
|
||||||
|
}]);
|
||||||
|
assertEquals(projection.workerState, paused);
|
||||||
|
assert(
|
||||||
|
projection.lines.some((line) =>
|
||||||
|
line.eventId === "conflict:worker-state-conflict" && line.error
|
||||||
|
),
|
||||||
|
"conflicting equal-version snapshots must fail closed",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
Deno.test("snapshot replaces a live error with one durable run_errored row", () => {
|
Deno.test("snapshot replaces a live error with one durable run_errored row", () => {
|
||||||
const projector = createConsoleProjector();
|
const projector = createConsoleProjector();
|
||||||
let projection = projector.append([
|
let projection = projector.append([
|
||||||
@@ -213,7 +290,7 @@ Deno.test("snapshot replaces a live error with one durable run_errored row", ()
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
eventId: "idle-after-error",
|
eventId: "idle-after-error",
|
||||||
event: { event: "status", data: { status: "idle" } } satisfies Event,
|
event: { event: "worker_state", data: { snapshot: workerState("idle") } } satisfies Event,
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
@@ -653,7 +730,7 @@ Deno.test("projectConsole streams distinct Bash stdout and stderr through termin
|
|||||||
Deno.test("snapshot restores bounded in-flight Bash command output", () => {
|
Deno.test("snapshot restores bounded in-flight Bash command output", () => {
|
||||||
const snapshot = snapshotEvent("/repo");
|
const snapshot = snapshotEvent("/repo");
|
||||||
if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected");
|
if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected");
|
||||||
snapshot.data.status = "running";
|
snapshot.data.state = workerState("running");
|
||||||
snapshot.data.in_flight = {
|
snapshot.data.in_flight = {
|
||||||
blocks: [{
|
blocks: [{
|
||||||
kind: "tool_call",
|
kind: "tool_call",
|
||||||
@@ -1403,7 +1480,7 @@ Deno.test("projectConsole hides lifecycle events and renders system items", () =
|
|||||||
const projection = projectConsole([
|
const projection = projectConsole([
|
||||||
{
|
{
|
||||||
eventId: "30",
|
eventId: "30",
|
||||||
event: { event: "status", data: { status: "running" } } satisfies Event,
|
event: { event: "worker_state", data: { snapshot: workerState("running") } } satisfies Event,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
eventId: "31",
|
eventId: "31",
|
||||||
@@ -1527,7 +1604,7 @@ Deno.test("projectConsole renders snapshot entries and in-flight output", () =>
|
|||||||
context_window: 100,
|
context_window: 100,
|
||||||
context_tokens: 20,
|
context_tokens: 20,
|
||||||
},
|
},
|
||||||
status: "running",
|
state: workerState("running"),
|
||||||
in_flight: {
|
in_flight: {
|
||||||
blocks: [
|
blocks: [
|
||||||
{ kind: "text", text: "partial" },
|
{ kind: "text", text: "partial" },
|
||||||
@@ -1578,7 +1655,7 @@ Deno.test("projectConsole restores system items from snapshot entries", () => {
|
|||||||
context_window: 100,
|
context_window: 100,
|
||||||
context_tokens: 20,
|
context_tokens: 20,
|
||||||
},
|
},
|
||||||
status: "idle",
|
state: workerState("idle"),
|
||||||
},
|
},
|
||||||
} satisfies Event,
|
} satisfies Event,
|
||||||
}]);
|
}]);
|
||||||
@@ -1922,7 +1999,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
|
|||||||
kind: "sub_worker",
|
kind: "sub_worker",
|
||||||
},
|
},
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: { event: "status", data: { status: "running" } },
|
event: { event: "worker_state", data: { snapshot: workerState("running") } },
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -1941,7 +2018,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
|
|||||||
kind: "sub_worker",
|
kind: "sub_worker",
|
||||||
},
|
},
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: { event: "status", data: { status: "idle" } },
|
event: { event: "worker_state", data: { snapshot: workerState("idle") } },
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}]);
|
}]);
|
||||||
@@ -2033,7 +2110,7 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections"
|
|||||||
kind: "sub_worker",
|
kind: "sub_worker",
|
||||||
},
|
},
|
||||||
revision: 1,
|
revision: 1,
|
||||||
event: { event: "status", data: { status: "running" } },
|
event: { event: "worker_state", data: { snapshot: workerState("running") } },
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}]);
|
}]);
|
||||||
|
|||||||
@@ -10,6 +10,9 @@ import type {
|
|||||||
InternalWorkerRef,
|
InternalWorkerRef,
|
||||||
InternalWorkerSnapshot,
|
InternalWorkerSnapshot,
|
||||||
Segment,
|
Segment,
|
||||||
|
WorkerState,
|
||||||
|
WorkerStateSnapshot,
|
||||||
|
WorkerStatus,
|
||||||
} from "$lib/generated/protocol";
|
} from "$lib/generated/protocol";
|
||||||
import { stringify as stringifyYaml } from "yaml";
|
import { stringify as stringifyYaml } from "yaml";
|
||||||
import { workspaceRoute } from "$lib/workspace/api/http";
|
import { workspaceRoute } from "$lib/workspace/api/http";
|
||||||
@@ -169,6 +172,7 @@ export type ConsoleProjection = {
|
|||||||
tasks: ConsoleTask[];
|
tasks: ConsoleTask[];
|
||||||
taskNextId: number;
|
taskNextId: number;
|
||||||
status: string | null;
|
status: string | null;
|
||||||
|
workerState: WorkerStateSnapshot | null;
|
||||||
usage: string | null;
|
usage: string | null;
|
||||||
runActivity: RunActivityStats;
|
runActivity: RunActivityStats;
|
||||||
cwd: string | null;
|
cwd: string | null;
|
||||||
@@ -251,12 +255,22 @@ export function isConsoleProjectionEvent(event: ProtocolEvent): boolean {
|
|||||||
return event.event !== "completions";
|
return event.event !== "completions";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function workerStatusFromState(snapshot: WorkerStateSnapshot): WorkerStatus {
|
||||||
|
if (snapshot.state.kind === "idle") return "idle";
|
||||||
|
if (
|
||||||
|
snapshot.state.state.kind === "run" &&
|
||||||
|
snapshot.state.state.state === "paused"
|
||||||
|
) return "paused";
|
||||||
|
return "running";
|
||||||
|
}
|
||||||
|
|
||||||
export function emptyConsoleProjection(): ConsoleProjection {
|
export function emptyConsoleProjection(): ConsoleProjection {
|
||||||
return {
|
return {
|
||||||
lines: [],
|
lines: [],
|
||||||
tasks: [],
|
tasks: [],
|
||||||
taskNextId: 1,
|
taskNextId: 1,
|
||||||
status: null,
|
status: null,
|
||||||
|
workerState: null,
|
||||||
usage: null,
|
usage: null,
|
||||||
runActivity: emptyRunActivityStats(),
|
runActivity: emptyRunActivityStats(),
|
||||||
cwd: null,
|
cwd: null,
|
||||||
@@ -783,6 +797,60 @@ function refreshCompactionActivity(
|
|||||||
return changed ? { ...projection, lines } : projection;
|
return changed ? { ...projection, lines } : projection;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
|
||||||
|
if (left.kind !== right.kind) return false;
|
||||||
|
if (left.kind === "idle" || right.kind === "idle") return true;
|
||||||
|
return left.state.kind === right.state.kind &&
|
||||||
|
left.state.state === right.state.state;
|
||||||
|
}
|
||||||
|
|
||||||
|
function workerStateSnapshotEqual(
|
||||||
|
left: WorkerStateSnapshot,
|
||||||
|
right: WorkerStateSnapshot,
|
||||||
|
): boolean {
|
||||||
|
return left.execution_generation === right.execution_generation &&
|
||||||
|
left.revision === right.revision &&
|
||||||
|
left.last_command_id === right.last_command_id &&
|
||||||
|
workerStateEqual(left.state, right.state);
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyWorkerStateSnapshot(
|
||||||
|
projection: ConsoleProjection,
|
||||||
|
incoming: WorkerStateSnapshot,
|
||||||
|
eventId: string,
|
||||||
|
): void {
|
||||||
|
const current = projection.workerState;
|
||||||
|
if (!current) {
|
||||||
|
projection.workerState = incoming;
|
||||||
|
projection.status = workerStatusFromState(incoming);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const generationOrder = incoming.execution_generation -
|
||||||
|
current.execution_generation;
|
||||||
|
const revisionOrder = incoming.revision - current.revision;
|
||||||
|
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
|
||||||
|
projection.workerState = incoming;
|
||||||
|
projection.status = workerStatusFromState(incoming);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!workerStateSnapshotEqual(current, incoming)) {
|
||||||
|
projection.lines.push(
|
||||||
|
line(
|
||||||
|
`${eventId}:worker-state-conflict`,
|
||||||
|
"error",
|
||||||
|
"error · internal",
|
||||||
|
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
|
||||||
|
undefined,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function applyProtocolEvent(
|
export function applyProtocolEvent(
|
||||||
projection: ConsoleProjection,
|
projection: ConsoleProjection,
|
||||||
envelope: ConsoleEventInput,
|
envelope: ConsoleEventInput,
|
||||||
@@ -793,6 +861,7 @@ export function applyProtocolEvent(
|
|||||||
tasks: [...projection.tasks],
|
tasks: [...projection.tasks],
|
||||||
taskNextId: projection.taskNextId,
|
taskNextId: projection.taskNextId,
|
||||||
status: projection.status,
|
status: projection.status,
|
||||||
|
workerState: projection.workerState,
|
||||||
usage: projection.usage,
|
usage: projection.usage,
|
||||||
runActivity: applyRunActivityEvent(
|
runActivity: applyRunActivityEvent(
|
||||||
projection.runActivity,
|
projection.runActivity,
|
||||||
@@ -903,7 +972,6 @@ export function applyProtocolEvent(
|
|||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
case "snapshot": {
|
case "snapshot": {
|
||||||
next.status = event.data.status;
|
|
||||||
next.cwd = event.data.greeting.cwd;
|
next.cwd = event.data.greeting.cwd;
|
||||||
const snapshot = snapshotProjectionFromSession(
|
const snapshot = snapshotProjectionFromSession(
|
||||||
envelope.eventId,
|
envelope.eventId,
|
||||||
@@ -953,6 +1021,7 @@ export function applyProtocolEvent(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "internal_worker": {
|
case "internal_worker": {
|
||||||
@@ -1000,8 +1069,15 @@ export function applyProtocolEvent(
|
|||||||
if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1);
|
if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "status":
|
case "worker_state":
|
||||||
next.status = event.data.status;
|
applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
|
||||||
|
break;
|
||||||
|
case "command_acknowledged":
|
||||||
|
applyWorkerStateSnapshot(
|
||||||
|
next,
|
||||||
|
event.data.acknowledgement.state,
|
||||||
|
envelope.eventId,
|
||||||
|
);
|
||||||
break;
|
break;
|
||||||
case "command":
|
case "command":
|
||||||
applyCommandEvent(next, envelope.eventId, event.data.event);
|
applyCommandEvent(next, envelope.eventId, event.data.event);
|
||||||
@@ -1939,6 +2015,7 @@ function snapshotProjectionFromSession(
|
|||||||
tasks: [],
|
tasks: [],
|
||||||
taskNextId: 1,
|
taskNextId: 1,
|
||||||
status: null,
|
status: null,
|
||||||
|
workerState: null,
|
||||||
usage: null,
|
usage: null,
|
||||||
runActivity: emptyRunActivityStats(),
|
runActivity: emptyRunActivityStats(),
|
||||||
cwd,
|
cwd,
|
||||||
|
|||||||
@@ -75,7 +75,12 @@ Deno.test("new invoke and running snapshot reset run activity", () => {
|
|||||||
data: {
|
data: {
|
||||||
entries: [],
|
entries: [],
|
||||||
greeting: { text: "", profile: "" },
|
greeting: { text: "", profile: "" },
|
||||||
status: "idle",
|
state: {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 0,
|
||||||
|
last_command_id: 0,
|
||||||
|
state: { kind: "idle" },
|
||||||
|
},
|
||||||
in_flight: {},
|
in_flight: {},
|
||||||
internal_workers: [],
|
internal_workers: [],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -25,7 +25,9 @@ export function applyRunActivityEvent(
|
|||||||
case "invoke_start":
|
case "invoke_start":
|
||||||
return { ...emptyRunActivityStats(), startedAtMs: observedAtMs };
|
return { ...emptyRunActivityStats(), startedAtMs: observedAtMs };
|
||||||
case "snapshot":
|
case "snapshot":
|
||||||
return event.data.status === "running"
|
return event.data.state.state.kind === "busy" &&
|
||||||
|
!(event.data.state.state.state.kind === "run" &&
|
||||||
|
event.data.state.state.state.state === "paused")
|
||||||
? { ...emptyRunActivityStats(), startedAtMs: observedAtMs }
|
? { ...emptyRunActivityStats(), startedAtMs: observedAtMs }
|
||||||
: emptyRunActivityStats();
|
: emptyRunActivityStats();
|
||||||
case "turn_start":
|
case "turn_start":
|
||||||
|
|||||||
@@ -620,7 +620,7 @@ Deno.test("Worker Console paste chips preserve typed draft and target authority"
|
|||||||
consolePage.includes("preserveExactText: value.textPastes.length > 0") &&
|
consolePage.includes("preserveExactText: value.textPastes.length > 0") &&
|
||||||
consolePage.includes("composerDrafts.set(activeComposerTargetKey") &&
|
consolePage.includes("composerDrafts.set(activeComposerTargetKey") &&
|
||||||
consolePage.includes("switchComposerTarget(target)") &&
|
consolePage.includes("switchComposerTarget(target)") &&
|
||||||
consolePage.includes('sendControl({ method: "cancel" }, "Stop")'),
|
consolePage.includes('sendWorkerControl("cancel")'),
|
||||||
"Paste chips should use shared threshold classification, atomic keyboard behavior, accessible labels, typed restore, and per-Worker draft authority",
|
"Paste chips should use shared threshold classification, atomic keyboard behavior, accessible labels, typed restore, and per-Worker draft authority",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -787,7 +787,10 @@ Deno.test("Worker Console page is routed by runtime_id and worker_id through bac
|
|||||||
consolePage.includes(
|
consolePage.includes(
|
||||||
'const composerEditable = $derived(protocolState === "open" && !sending);',
|
'const composerEditable = $derived(protocolState === "open" && !sending);',
|
||||||
) &&
|
) &&
|
||||||
consolePage.includes('sendControl({ method: "cancel" }, "Stop")') &&
|
consolePage.includes('sendWorkerControl("cancel")') &&
|
||||||
|
consolePage.includes("lifecycleMethod(command)") &&
|
||||||
|
consolePage.includes("expected_worker_state_revision") &&
|
||||||
|
consolePage.includes("expected_execution_generation") &&
|
||||||
consolePage.includes("onsubmit={handleComposerSubmit}") &&
|
consolePage.includes("onsubmit={handleComposerSubmit}") &&
|
||||||
consolePage.includes("disabled={!composerEditable}") &&
|
consolePage.includes("disabled={!composerEditable}") &&
|
||||||
consolePage.includes("class:stop={workerRunning}") &&
|
consolePage.includes("class:stop={workerRunning}") &&
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import type {
|
|||||||
Event as PodProtocolEvent,
|
Event as PodProtocolEvent,
|
||||||
Method as PodProtocolMethod,
|
Method as PodProtocolMethod,
|
||||||
Segment as PodProtocolSegment,
|
Segment as PodProtocolSegment,
|
||||||
|
WorkerStateSnapshot,
|
||||||
} from "$lib/generated/protocol";
|
} from "$lib/generated/protocol";
|
||||||
import type {
|
import type {
|
||||||
GitCommitSummary as SharedGitCommitSummary,
|
GitCommitSummary as SharedGitCommitSummary,
|
||||||
@@ -99,6 +100,7 @@ export type Worker = {
|
|||||||
tags: string[];
|
tags: string[];
|
||||||
workspace: { visibility: string; identity: string };
|
workspace: { visibility: string; identity: string };
|
||||||
state: string;
|
state: string;
|
||||||
|
worker_state?: WorkerStateSnapshot | null;
|
||||||
pinned?: boolean;
|
pinned?: boolean;
|
||||||
retention_state?: string;
|
retention_state?: string;
|
||||||
last_seen_at?: string | null;
|
last_seen_at?: string | null;
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import type { WorkerStateSnapshot } from "$lib/generated/protocol";
|
||||||
|
|
||||||
|
export function liveWorkerState(worker: {
|
||||||
|
state: string;
|
||||||
|
worker_state?: WorkerStateSnapshot | null;
|
||||||
|
}): string {
|
||||||
|
const state = worker.worker_state?.state;
|
||||||
|
if (!state) return worker.state === "stopped" ? "stopped" : "unknown";
|
||||||
|
if (state.kind === "idle") return "idle";
|
||||||
|
if (state.state.kind === "maintenance") return "running";
|
||||||
|
return state.state.state === "paused" ? "paused" : "running";
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void {
|
|||||||
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
import { liveWorkerState } from './worker-state';
|
||||||
import {
|
import {
|
||||||
applyWorkspaceWorkersFrame,
|
applyWorkspaceWorkersFrame,
|
||||||
createWorkspaceWorkersProjection,
|
createWorkspaceWorkersProjection,
|
||||||
@@ -33,6 +34,22 @@ function worker(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
|
||||||
|
const active = worker('runtime-a', 'worker-1', 1);
|
||||||
|
active.worker_state = {
|
||||||
|
execution_generation: 4,
|
||||||
|
revision: 2,
|
||||||
|
last_command_id: 1,
|
||||||
|
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
|
||||||
|
};
|
||||||
|
assertEquals(liveWorkerState(active), 'paused');
|
||||||
|
|
||||||
|
const unavailable = worker('runtime-a', 'worker-2', 1);
|
||||||
|
assertEquals(liveWorkerState(unavailable), 'unknown');
|
||||||
|
unavailable.state = 'stopped';
|
||||||
|
assertEquals(liveWorkerState(unavailable), 'stopped');
|
||||||
|
});
|
||||||
|
|
||||||
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
|
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
|
||||||
const projection = createWorkspaceWorkersProjection();
|
const projection = createWorkspaceWorkersProjection();
|
||||||
const frame: SubscriptionFrame = {
|
const frame: SubscriptionFrame = {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
applyWorkspaceWorkersFrame,
|
applyWorkspaceWorkersFrame,
|
||||||
createWorkspaceWorkersProjection,
|
createWorkspaceWorkersProjection,
|
||||||
} from './worker-subscription-model';
|
} from './worker-subscription-model';
|
||||||
|
import { liveWorkerState } from './worker-state';
|
||||||
import { compareWorkersForSidebar } from './workers';
|
import { compareWorkersForSidebar } from './workers';
|
||||||
import type { Worker } from './types';
|
import type { Worker } from './types';
|
||||||
|
|
||||||
@@ -90,7 +91,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker {
|
|||||||
profile: worker.profile ?? null,
|
profile: worker.profile ?? null,
|
||||||
tags: [],
|
tags: [],
|
||||||
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
|
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
|
||||||
state: worker.state,
|
state: liveWorkerState(worker),
|
||||||
|
worker_state: worker.worker_state,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: 'transient',
|
retention_state: 'transient',
|
||||||
implementation: {
|
implementation: {
|
||||||
|
|||||||
+60
-31
@@ -52,11 +52,7 @@
|
|||||||
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
|
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
|
||||||
import { workspaceApiPath } from "$lib/workspace/api/http";
|
import { workspaceApiPath } from "$lib/workspace/api/http";
|
||||||
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
|
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
|
||||||
import type {
|
import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types";
|
||||||
Diagnostic,
|
|
||||||
Worker,
|
|
||||||
PodProtocolEvent,
|
|
||||||
} from "$lib/workspace/sidebar/types";
|
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
data: {
|
data: {
|
||||||
@@ -207,7 +203,6 @@
|
|||||||
);
|
);
|
||||||
let pendingObservationEvents: ConsoleEventInput[] = [];
|
let pendingObservationEvents: ConsoleEventInput[] = [];
|
||||||
let protocolEventSequence = 0;
|
let protocolEventSequence = 0;
|
||||||
let pendingObservedStates: Array<string | null> = [];
|
|
||||||
let pendingStreamDiagnostics: Diagnostic[] = [];
|
let pendingStreamDiagnostics: Diagnostic[] = [];
|
||||||
let observationFlushHandle: number | null = null;
|
let observationFlushHandle: number | null = null;
|
||||||
let nextReloadToken = 0;
|
let nextReloadToken = 0;
|
||||||
@@ -249,7 +244,9 @@
|
|||||||
const diagnostics = $derived(
|
const diagnostics = $derived(
|
||||||
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
|
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
|
||||||
);
|
);
|
||||||
const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading");
|
const workerState = $derived(
|
||||||
|
liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"),
|
||||||
|
);
|
||||||
const workerRunning = $derived(workerState === "running");
|
const workerRunning = $derived(workerState === "running");
|
||||||
const workerPaused = $derived(workerState === "paused");
|
const workerPaused = $derived(workerState === "paused");
|
||||||
const composerEditable = $derived(protocolState === "open" && !sending);
|
const composerEditable = $derived(protocolState === "open" && !sending);
|
||||||
@@ -343,7 +340,6 @@
|
|||||||
observationFlushHandle = null;
|
observationFlushHandle = null;
|
||||||
}
|
}
|
||||||
pendingObservationEvents = [];
|
pendingObservationEvents = [];
|
||||||
pendingObservedStates = [];
|
|
||||||
pendingStreamDiagnostics = [];
|
pendingStreamDiagnostics = [];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -359,18 +355,15 @@
|
|||||||
function flushObservationBatch() {
|
function flushObservationBatch() {
|
||||||
observationFlushHandle = null;
|
observationFlushHandle = null;
|
||||||
const eventBatch = pendingObservationEvents;
|
const eventBatch = pendingObservationEvents;
|
||||||
const stateBatch = pendingObservedStates;
|
|
||||||
const diagnosticBatch = pendingStreamDiagnostics;
|
const diagnosticBatch = pendingStreamDiagnostics;
|
||||||
pendingObservationEvents = [];
|
pendingObservationEvents = [];
|
||||||
pendingObservedStates = [];
|
|
||||||
pendingStreamDiagnostics = [];
|
pendingStreamDiagnostics = [];
|
||||||
|
|
||||||
if (eventBatch.length > 0) {
|
if (eventBatch.length > 0) {
|
||||||
const latestState = stateBatch.findLast((state) => state !== null);
|
|
||||||
if (latestState) {
|
|
||||||
liveWorkerState = latestState;
|
|
||||||
}
|
|
||||||
consoleProjection = consoleProjector.append(eventBatch);
|
consoleProjection = consoleProjector.append(eventBatch);
|
||||||
|
liveWorkerState = consoleProjection.status === "shutdown"
|
||||||
|
? "shutdown"
|
||||||
|
: workerStateFromSnapshot(consoleProjection.workerState);
|
||||||
advanceEventObservedAtVersion();
|
advanceEventObservedAtVersion();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -407,7 +400,6 @@
|
|||||||
event: payload,
|
event: payload,
|
||||||
observedAtMs,
|
observedAtMs,
|
||||||
});
|
});
|
||||||
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
|
|
||||||
scheduleObservationFlush();
|
scheduleObservationFlush();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -541,9 +533,42 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let nextWorkerCommandId = 1;
|
||||||
|
|
||||||
|
function lifecycleMethod(
|
||||||
|
command: "pause" | "cancel" | "resume" | "compact",
|
||||||
|
): ProtocolMethod | null {
|
||||||
|
const state = consoleProjection.workerState;
|
||||||
|
if (!state) {
|
||||||
|
sendError = "Worker state snapshot is not available; reconnect before sending control.";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const commandId = Math.max(
|
||||||
|
nextWorkerCommandId,
|
||||||
|
state.last_command_id + 1,
|
||||||
|
);
|
||||||
|
nextWorkerCommandId = commandId + 1;
|
||||||
|
const envelope = {
|
||||||
|
command_id: commandId,
|
||||||
|
expected_execution_generation: state.execution_generation,
|
||||||
|
expected_worker_state_revision: state.revision,
|
||||||
|
};
|
||||||
|
switch (command) {
|
||||||
|
case "pause":
|
||||||
|
return { method: "pause", params: { command: envelope } };
|
||||||
|
case "cancel":
|
||||||
|
return { method: "cancel", params: { command: envelope } };
|
||||||
|
case "resume":
|
||||||
|
return { method: "resume", params: { command: envelope } };
|
||||||
|
case "compact":
|
||||||
|
return { method: "compact", params: { command: envelope } };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function sendWorkerControl(command: "pause" | "cancel" | "resume") {
|
function sendWorkerControl(command: "pause" | "cancel" | "resume") {
|
||||||
const label = command[0].toUpperCase() + command.slice(1);
|
const label = command[0].toUpperCase() + command.slice(1);
|
||||||
sendControl({ method: command }, label);
|
const method = lifecycleMethod(command);
|
||||||
|
if (method) sendControl(method, label);
|
||||||
}
|
}
|
||||||
|
|
||||||
function isEditableTarget(target: EventTarget | null): boolean {
|
function isEditableTarget(target: EventTarget | null): boolean {
|
||||||
@@ -627,8 +652,11 @@
|
|||||||
auto_run: true,
|
auto_run: true,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
case "compact":
|
case "compact": {
|
||||||
return { method: "compact" };
|
const method = lifecycleMethod("compact");
|
||||||
|
if (!method) throw new Error("Worker state snapshot is not available");
|
||||||
|
return method;
|
||||||
|
}
|
||||||
case "list_rewind_targets":
|
case "list_rewind_targets":
|
||||||
return { method: "list_rewind_targets" };
|
return { method: "list_rewind_targets" };
|
||||||
case "register_peer":
|
case "register_peer":
|
||||||
@@ -691,7 +719,7 @@
|
|||||||
|
|
||||||
function handleComposerSubmit() {
|
function handleComposerSubmit() {
|
||||||
if (workerRunning) {
|
if (workerRunning) {
|
||||||
sendControl({ method: "cancel" }, "Stop");
|
sendWorkerControl("cancel");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
void submitDraft(composerInputElement?.snapshot() ?? draft);
|
void submitDraft(composerInputElement?.snapshot() ?? draft);
|
||||||
@@ -889,18 +917,16 @@
|
|||||||
handleComposerSubmit();
|
handleComposerSubmit();
|
||||||
}
|
}
|
||||||
|
|
||||||
function workerStateFromProtocolEvent(
|
function workerStateFromSnapshot(
|
||||||
event: PodProtocolEvent,
|
snapshot: ConsoleProjection["workerState"],
|
||||||
): string | null {
|
): string | null {
|
||||||
switch (event.event) {
|
if (!snapshot) return null;
|
||||||
case "snapshot":
|
return snapshot.state.kind === "idle"
|
||||||
case "status":
|
? "idle"
|
||||||
return event.data.status;
|
: snapshot.state.state.kind === "run" &&
|
||||||
case "shutdown":
|
snapshot.state.state.state === "paused"
|
||||||
return "shutdown";
|
? "paused"
|
||||||
default:
|
: "running";
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function connectProtocolTransport(
|
function connectProtocolTransport(
|
||||||
@@ -1620,7 +1646,10 @@
|
|||||||
type="button"
|
type="button"
|
||||||
class="secondary-button"
|
class="secondary-button"
|
||||||
disabled={protocolState !== "open"}
|
disabled={protocolState !== "open"}
|
||||||
onclick={() => sendControl({ method: "compact" }, "Compact")}
|
onclick={() => {
|
||||||
|
const method = lifecycleMethod("compact");
|
||||||
|
if (method) sendControl(method, "Compact");
|
||||||
|
}}
|
||||||
>
|
>
|
||||||
Compact
|
Compact
|
||||||
</button>
|
</button>
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
import { workerHref } from '$lib/workspace/resource-links';
|
import { workerHref } from '$lib/workspace/resource-links';
|
||||||
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
|
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
|
||||||
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
|
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
|
||||||
|
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||||
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
|
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
|
||||||
import type { PageProps } from './$types';
|
import type { PageProps } from './$types';
|
||||||
|
|
||||||
@@ -136,7 +137,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
function workerStatus(worker: Worker): string {
|
function workerStatus(worker: Worker): string {
|
||||||
return worker.state;
|
return liveWorkerState(worker);
|
||||||
}
|
}
|
||||||
|
|
||||||
function workerProfile(worker: Worker): string {
|
function workerProfile(worker: Worker): string {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { workspaceRoute } from '$lib/workspace/api/http';
|
import { workspaceRoute } from '$lib/workspace/api/http';
|
||||||
|
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||||
import type { PageData } from './$types';
|
import type { PageData } from './$types';
|
||||||
let { data }: { data: PageData } = $props();
|
let { data }: { data: PageData } = $props();
|
||||||
</script>
|
</script>
|
||||||
@@ -24,7 +25,7 @@
|
|||||||
>Open console</a>
|
>Open console</a>
|
||||||
</header>
|
</header>
|
||||||
<dl class="resource-meta">
|
<dl class="resource-meta">
|
||||||
<dt>Status</dt><dd>{data.worker.state}</dd>
|
<dt>Status</dt><dd>{liveWorkerState(data.worker)}</dd>
|
||||||
<dt>Profile</dt><dd>{data.worker.profile}</dd>
|
<dt>Profile</dt><dd>{data.worker.profile}</dd>
|
||||||
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
|
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
|
||||||
</dl>
|
</dl>
|
||||||
|
|||||||
Reference in New Issue
Block a user