fix: separate worker state from runtime lifecycle

This commit is contained in:
2026-09-06 10:05:54 +09:00
parent 282a8d31b5
commit 2d1956b653
21 changed files with 602 additions and 188 deletions
+1
View File
@@ -125,6 +125,7 @@ pub enum WorkerCommandDisposition {
StaleExecutionGeneration, StaleExecutionGeneration,
StaleWorkerStateRevision, StaleWorkerStateRevision,
StaleCommandId, StaleCommandId,
Conflict,
InvalidState, InvalidState,
} }
+6
View File
@@ -573,6 +573,11 @@ pub struct SubscriptionWorker {
pub resource_key: Option<String>, pub resource_key: Option<String>,
/// Producer-owned monotonic revision for this Worker subject. /// Producer-owned monotonic revision for this Worker subject.
pub subject_revision: u64, pub subject_revision: u64,
/// Latest revisioned foreground state observed from the Worker. This remains
/// absent until an authoritative Worker snapshot/event has been applied.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<crate::WorkerStateSnapshot>,
/// Runtime catalog lifecycle compatibility projection; not foreground-state authority.
pub state: SubscriptionWorkerState, pub state: SubscriptionWorkerState,
#[serde(default)] #[serde(default)]
pub has_running_internal_workers: bool, pub has_running_internal_workers: bool,
@@ -874,6 +879,7 @@ mod tests {
runtime_id: None, runtime_id: None,
resource_key: None, resource_key: None,
subject_revision: 0, subject_revision: 0,
worker_state: None,
state: SubscriptionWorkerState::Idle, state: SubscriptionWorkerState::Idle,
has_running_internal_workers: false, has_running_internal_workers: false,
workspace_id: Some("workspace-1".to_string()), workspace_id: Some("workspace-1".to_string()),
+37 -16
View File
@@ -1180,18 +1180,14 @@ impl App {
self.assistant_streaming = false; self.assistant_streaming = false;
} }
Event::TurnStart { .. } => { Event::TurnStart { .. } => {
self.set_worker_status(WorkerStatus::Running);
self.run_requests += 1; self.run_requests += 1;
self.current_tool = None; self.current_tool = None;
self.latest_llm_wait_event = None; self.latest_llm_wait_event = None;
self.assistant_streaming = false; self.assistant_streaming = false;
} }
Event::InvokeStart { .. } => { Event::InvokeStart { .. } => {}
self.set_worker_status(WorkerStatus::Running);
}
// UI consumers of per-attempt LlmCall semantics remain out of scope; // UI consumers of per-attempt LlmCall semantics remain out of scope;
// the run-level status starts at InvokeStart and TurnStart counts each // authoritative run state comes only from WorkerStateSnapshot.
// LLM request within that run.
Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => { Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => {
self.latest_llm_wait_event = None; self.latest_llm_wait_event = None;
} }
@@ -1398,12 +1394,7 @@ impl App {
output_tokens: self.run_output_tokens, output_tokens: self.run_output_tokens,
}); });
self.pending_submit_rollback = None; self.pending_submit_rollback = None;
self.reset_run_state(match result { self.reset_run_state();
RunResult::Paused => WorkerStatus::Paused,
RunResult::Finished | RunResult::LimitReached | RunResult::RolledBack => {
WorkerStatus::Idle
}
});
} }
} }
Event::CompactStart { .. } => { Event::CompactStart { .. } => {
@@ -1536,7 +1527,7 @@ impl App {
}; };
self.completion = None; self.completion = None;
self.close_rewind_picker(); self.close_rewind_picker();
self.reset_run_state(self.worker_status); self.reset_run_state();
let mut message = if restored_composer { let mut message = if restored_composer {
format!( format!(
"Rewound session: discarded {} log entries; restored selected input to composer.", "Rewound session: discarded {} log entries; restored selected input to composer.",
@@ -1584,8 +1575,7 @@ impl App {
None None
} }
fn reset_run_state(&mut self, status: WorkerStatus) { fn reset_run_state(&mut self) {
self.set_worker_status(status);
self.run_requests = 0; self.run_requests = 0;
self.run_upload_tokens = 0; self.run_upload_tokens = 0;
self.run_output_tokens = 0; self.run_output_tokens = 0;
@@ -1615,7 +1605,7 @@ impl App {
"Rolled back empty assistant turn; no local submitted input was available to restore." "Rolled back empty assistant turn; no local submitted input was available to restore."
.to_owned() .to_owned()
}; };
self.reset_run_state(WorkerStatus::Idle); self.reset_run_state();
self.blocks.push(Block::Alert { self.blocks.push(Block::Alert {
level: AlertLevel::Warn, level: AlertLevel::Warn,
source: AlertSource::Worker, source: AlertSource::Worker,
@@ -3583,6 +3573,37 @@ mod completion_flow_tests {
assert!(matches!(app.blocks.first(), Some(Block::Greeting(_)))); assert!(matches!(app.blocks.first(), Some(Block::Greeting(_))));
} }
#[test]
fn occurrence_events_do_not_infer_foreground_worker_state() {
let mut app = App::new("test".into());
app.handle_worker_event(Event::TurnStart { turn: 1 });
app.handle_worker_event(Event::InvokeStart {
kind: protocol::InvokeKind::UserSend,
});
app.handle_worker_event(Event::RunEnd {
result: RunResult::Paused,
});
assert_eq!(app.worker_state.state, protocol::WorkerState::Idle);
assert_eq!(app.worker_status, WorkerStatus::Idle);
let running = WorkerStateSnapshot {
execution_generation: 1,
revision: 1,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 0,
};
app.handle_worker_event(Event::WorkerState {
snapshot: running.clone(),
});
app.handle_worker_event(Event::RunEnd {
result: RunResult::Finished,
});
assert_eq!(app.worker_state, running);
assert_eq!(app.worker_status, WorkerStatus::Running);
}
#[test] #[test]
fn worker_state_events_and_acknowledgements_share_monotonic_application() { fn worker_state_events_and_acknowledgements_share_monotonic_application() {
let mut app = App::new("test".into()); let mut app = App::new("test".into());
+31 -3
View File
@@ -321,7 +321,7 @@ fn row_line(
Span::raw(" "), Span::raw(" "),
Span::styled( Span::styled(
pad_column(&worker_state(worker), widths.state), pad_column(&worker_state(worker), widths.state),
state_style(worker.state.as_str()), state_style(worker_state_label(worker)),
), ),
Span::raw(" "), Span::raw(" "),
Span::styled( Span::styled(
@@ -341,8 +341,20 @@ fn worker_name(worker: &BackendWorkerSummary) -> &str {
} }
} }
fn worker_state_label(worker: &BackendWorkerSummary) -> &str {
match worker.worker_state.as_ref().map(|state| &state.state) {
Some(protocol::WorkerState::Idle) => "idle",
Some(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Paused,
))) => "paused",
Some(protocol::WorkerState::Busy(_)) => "running",
None if worker.state == "stopped" => "stopped",
None => "unknown",
}
}
fn worker_state(worker: &BackendWorkerSummary) -> String { fn worker_state(worker: &BackendWorkerSummary) -> String {
format!("[{}]", worker.state) format!("[{}]", worker_state_label(worker))
} }
fn text_width(value: &str) -> usize { fn text_width(value: &str) -> usize {
@@ -413,7 +425,15 @@ mod tests {
identity: "ws".to_string(), identity: "ws".to_string(),
workspace_id: Some("ws".to_string()), workspace_id: Some("ws".to_string()),
}, },
state: "running".to_string(), state: "idle".to_string(),
worker_state: Some(protocol::WorkerStateSnapshot {
execution_generation: 1,
revision: 1,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 0,
}),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: String::new(), retention_state: String::new(),
@@ -450,6 +470,7 @@ mod tests {
worker.display_name = "Coder".to_string(); worker.display_name = "Coder".to_string();
worker.label = "Coder · T-585".to_string(); worker.label = "Coder · T-585".to_string();
worker.state = "stopped".to_string(); worker.state = "stopped".to_string();
worker.worker_state = None;
worker.working_directory = Some( worker.working_directory = Some(
serde_json::from_value(serde_json::json!({ serde_json::from_value(serde_json::json!({
"working_directory_id": "001a06a9f0202000000", "working_directory_id": "001a06a9f0202000000",
@@ -478,12 +499,19 @@ mod tests {
short.label = "Coder".to_string(); short.label = "Coder".to_string();
short.display_name = short.label.clone(); short.display_name = short.label.clone();
short.state = "idle".to_string(); short.state = "idle".to_string();
short.worker_state = Some(protocol::WorkerStateSnapshot {
execution_generation: 1,
revision: 2,
state: protocol::WorkerState::Idle,
last_command_id: 0,
});
let mut long = worker("runtime-a", "worker-b", None); let mut long = worker("runtime-a", "worker-b", None);
long.resource_key = "W-100".to_string(); long.resource_key = "W-100".to_string();
long.label = "Longer worker · T-9".to_string(); long.label = "Longer worker · T-9".to_string();
long.display_name = long.label.clone(); long.display_name = long.label.clone();
long.state = "stopped".to_string(); long.state = "stopped".to_string();
long.worker_state = None;
for worker in [&mut short, &mut long] { for worker in [&mut short, &mut long] {
worker.working_directory = Some( worker.working_directory = Some(
+6
View File
@@ -307,6 +307,8 @@ pub struct WorkerSummary {
pub worker_id: WorkerId, pub worker_id: WorkerId,
pub status: WorkerStatus, pub status: WorkerStatus,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace_id: Option<String>, pub workspace_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>, pub working_directory: Option<WorkingDirectoryStatus>,
@@ -325,6 +327,8 @@ pub struct WorkerDetail {
pub worker_id: WorkerId, pub worker_id: WorkerId,
pub status: WorkerStatus, pub status: WorkerStatus,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace_id: Option<String>, pub workspace_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub working_directory: Option<WorkingDirectoryStatus>, pub working_directory: Option<WorkingDirectoryStatus>,
@@ -341,6 +345,8 @@ pub struct WorkerDetail {
pub struct WorkerLifecycleAck { pub struct WorkerLifecycleAck {
pub worker_ref: WorkerRef, pub worker_ref: WorkerRef,
pub status: WorkerStatus, pub status: WorkerStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
} }
#[cfg(test)] #[cfg(test)]
+10 -1
View File
@@ -3486,7 +3486,16 @@ mod ws_tests {
.. ..
}) if delivered_subscription_id == subscription_id }) if delivered_subscription_id == subscription_id
&& worker.worker_id.as_str() == worker_ref.worker_id.to_string() && worker.worker_id.as_str() == worker_ref.worker_id.to_string()
&& worker.state == protocol::subscription::SubscriptionWorkerState::Running && worker.state == protocol::subscription::SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running
)),
..
})
)
)); ));
let unsubscribe_request_id = let unsubscribe_request_id =
+131 -78
View File
@@ -700,6 +700,7 @@ impl Runtime {
worker_ref: worker_ref.clone(), worker_ref: worker_ref.clone(),
worker_id: worker_id.clone(), worker_id: worker_id.clone(),
status: WorkerStatus::Stopped, status: WorkerStatus::Stopped,
worker_state: None,
workspace_id: scope.map(|scope| scope.workspace_id.clone()), workspace_id: scope.map(|scope| scope.workspace_id.clone()),
request: durable_request, request: durable_request,
run_generation: 1, run_generation: 1,
@@ -787,7 +788,6 @@ impl Runtime {
let detail = self.commit_created_worker( let detail = self.commit_created_worker(
&worker_ref, &worker_ref,
handle, handle,
WorkerStatus::Running,
working_directory, working_directory,
dispatch_result, dispatch_result,
)?; )?;
@@ -797,7 +797,6 @@ impl Runtime {
self.commit_created_worker( self.commit_created_worker(
&worker_ref, &worker_ref,
handle, handle,
WorkerStatus::Idle,
working_directory, working_directory,
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
) )
@@ -1220,17 +1219,7 @@ impl Runtime {
state.ensure_running()?; state.ensure_running()?;
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
if let Some(snapshot) = dispatch_result.worker_state.as_ref() { if let Some(snapshot) = dispatch_result.worker_state.as_ref() {
worker.status = match snapshot.catalog_status() { let _ = worker.apply_worker_state(snapshot);
protocol::WorkerStatus::Idle => WorkerStatus::Idle,
protocol::WorkerStatus::Running => WorkerStatus::Running,
protocol::WorkerStatus::Paused => WorkerStatus::Paused,
protocol::WorkerStatus::Stopped => WorkerStatus::Stopped,
};
} else if matches!(
submission.as_ref().map(|ack| ack.disposition),
Some(protocol::SubmissionDisposition::Started)
) {
worker.status = WorkerStatus::Running;
} }
let status = worker.status; let status = worker.status;
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
@@ -1490,16 +1479,19 @@ impl Runtime {
&self, &self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
handle: WorkerExecutionHandle, handle: WorkerExecutionHandle,
status: WorkerStatus,
working_directory: Option<CatalogWorkingDirectoryStatus>, working_directory: Option<CatalogWorkingDirectoryStatus>,
_result: WorkerExecutionResult, result: WorkerExecutionResult,
) -> Result<WorkerDetail, RuntimeError> { ) -> Result<WorkerDetail, RuntimeError> {
let mut state = self.lock()?; let mut state = self.lock()?;
let detail = { let detail = {
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
worker.execution_handle = Some(handle); worker.execution_handle = Some(handle);
worker.execution_bound = true; worker.execution_bound = true;
worker.status = status; worker.status = WorkerStatus::Idle;
worker.worker_state = None;
if let Some(snapshot) = result.worker_state.as_ref() {
let _ = worker.apply_worker_state(snapshot);
}
worker.restore_intent = restore_intent_for_status(worker.status); worker.restore_intent = restore_intent_for_status(worker.status);
worker.working_directory = working_directory; worker.working_directory = working_directory;
worker.detail() worker.detail()
@@ -1536,16 +1528,14 @@ impl Runtime {
let Some(snapshot) = result.worker_state else { let Some(snapshot) = result.worker_state else {
return Ok(()); return Ok(());
}; };
let status = match snapshot.catalog_status() {
protocol::WorkerStatus::Idle => WorkerStatus::Idle,
protocol::WorkerStatus::Running => WorkerStatus::Running,
protocol::WorkerStatus::Paused => WorkerStatus::Paused,
protocol::WorkerStatus::Stopped => WorkerStatus::Stopped,
};
let mut state = self.lock()?; let mut state = self.lock()?;
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
worker.status = status; let applied = worker
worker.restore_intent = restore_intent_for_status(status); .apply_worker_state(&snapshot)
.is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied));
if !applied {
return Ok(());
}
state.publish_worker_upsert(worker_ref.worker_id)?; state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?; state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?; state.persist_worker(&worker_ref.worker_id)?;
@@ -1636,20 +1626,26 @@ impl Runtime {
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
reason: Option<String>, reason: Option<String>,
) -> Result<WorkerLifecycleAck, RuntimeError> { ) -> Result<WorkerLifecycleAck, RuntimeError> {
let current = { {
let state = self.lock()?; let state = self.lock()?;
state.ensure_running()?; state.ensure_running()?;
state.worker(worker_ref)?.status if state.worker(worker_ref)?.status == WorkerStatus::Stopped {
}; return Ok(WorkerLifecycleAck {
if matches!(current, WorkerStatus::Idle | WorkerStatus::Stopped) { worker_ref: worker_ref.clone(),
return Ok(WorkerLifecycleAck { status: WorkerStatus::Stopped,
worker_ref: worker_ref.clone(), worker_state: None,
status: current, });
}); }
} }
self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?; self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?;
let _ = reason; let _ = reason;
self.transition_worker_preserving_execution(worker_ref, WorkerStatus::Idle) let state = self.lock()?;
let worker = state.worker(worker_ref)?;
Ok(WorkerLifecycleAck {
worker_ref: worker_ref.clone(),
status: worker.status,
worker_state: worker.worker_state.clone(),
})
} }
/// Delete a non-running Worker through a workspace-scoped Runtime authorization context. /// Delete a non-running Worker through a workspace-scoped Runtime authorization context.
@@ -1795,12 +1791,13 @@ impl Runtime {
) -> Result<WorkerObservationEvent, RuntimeError> { ) -> Result<WorkerObservationEvent, RuntimeError> {
let mut state = self.lock()?; let mut state = self.lock()?;
state.ensure_worker_ref(worker_ref)?; state.ensure_worker_ref(worker_ref)?;
let status_changed = state.project_protocol_event_to_status(worker_ref, &payload); let worker_state_changed =
state.project_protocol_event_to_worker_state(worker_ref, &payload);
let activity_changed = state.project_internal_worker_activity(worker_ref, &payload); let activity_changed = state.project_internal_worker_activity(worker_ref, &payload);
if status_changed || activity_changed { if worker_state_changed || activity_changed {
state.publish_worker_upsert(worker_ref.worker_id)?; state.publish_worker_upsert(worker_ref.worker_id)?;
} }
if status_changed { if worker_state_changed {
state.persist_runtime_snapshot()?; state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?; state.persist_worker(&worker_ref.worker_id)?;
} }
@@ -1836,26 +1833,6 @@ impl Runtime {
Ok(()) Ok(())
} }
fn transition_worker_preserving_execution(
&self,
worker_ref: &WorkerRef,
status: WorkerStatus,
) -> Result<WorkerLifecycleAck, RuntimeError> {
let mut state = self.lock()?;
state.ensure_running()?;
let worker = state.worker_mut(worker_ref)?;
worker.status = status;
worker.restore_intent = restore_intent_for_status(status);
let status = worker.status;
state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?;
Ok(WorkerLifecycleAck {
worker_ref: worker_ref.clone(),
status,
})
}
fn transition_worker( fn transition_worker(
&self, &self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
@@ -1867,6 +1844,7 @@ impl Runtime {
let worker = state.worker_mut(worker_ref)?; let worker = state.worker_mut(worker_ref)?;
worker.status = status; worker.status = status;
worker.worker_state = None;
worker.restore_intent = restore_intent_for_status(status); worker.restore_intent = restore_intent_for_status(status);
worker.execution_handle = None; worker.execution_handle = None;
worker.internal_workers.clear(); worker.internal_workers.clear();
@@ -1877,6 +1855,7 @@ impl Runtime {
Ok(WorkerLifecycleAck { Ok(WorkerLifecycleAck {
worker_ref: worker_ref.clone(), worker_ref: worker_ref.clone(),
status, status,
worker_state: None,
}) })
} }
@@ -2301,6 +2280,7 @@ impl RuntimeState {
worker_ref: worker.worker_ref, worker_ref: worker.worker_ref,
worker_id: worker.worker_id, worker_id: worker.worker_id,
status: worker.status, status: worker.status,
worker_state: None,
workspace_id: worker.workspace_id, workspace_id: worker.workspace_id,
request: worker.request, request: worker.request,
run_generation, run_generation,
@@ -2630,6 +2610,7 @@ impl RuntimeState {
.get(&worker.worker_id) .get(&worker.worker_id)
.copied() .copied()
.unwrap_or(0), .unwrap_or(0),
worker_state: worker.worker_state.clone(),
state: subscription_worker_state(worker.status), state: subscription_worker_state(worker.status),
has_running_internal_workers: worker has_running_internal_workers: worker
.internal_workers .internal_workers
@@ -2965,7 +2946,7 @@ impl RuntimeState {
Self::update_internal_worker_activity(&mut worker.internal_workers, event) Self::update_internal_worker_activity(&mut worker.internal_workers, event)
} }
fn project_protocol_event_to_status( fn project_protocol_event_to_worker_state(
&mut self, &mut self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
event: &protocol::Event, event: &protocol::Event,
@@ -2973,7 +2954,7 @@ impl RuntimeState {
let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else { let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else {
return false; return false;
}; };
let next_status = match event { let incoming = match event {
protocol::Event::WorkerState { snapshot } protocol::Event::WorkerState { snapshot }
| protocol::Event::Snapshot { | protocol::Event::Snapshot {
state: snapshot, .. state: snapshot, ..
@@ -2983,21 +2964,16 @@ impl RuntimeState {
protocol::WorkerCommandAcknowledgement { protocol::WorkerCommandAcknowledgement {
state: snapshot, .. state: snapshot, ..
}, },
} => Some(match snapshot.catalog_status() { } => snapshot,
protocol::WorkerStatus::Idle => WorkerStatus::Idle, _ => return false,
protocol::WorkerStatus::Running => WorkerStatus::Running,
protocol::WorkerStatus::Paused => WorkerStatus::Paused,
protocol::WorkerStatus::Stopped => WorkerStatus::Stopped,
}),
_ => None,
}; };
if let Some(next_status) = next_status { match worker.apply_worker_state(incoming) {
let changed = worker.status != next_status; Ok(protocol::WorkerStateSnapshotApply::Applied) => true,
worker.status = next_status; Ok(
worker.restore_intent = restore_intent_for_status(next_status); protocol::WorkerStateSnapshotApply::Duplicate
changed | protocol::WorkerStateSnapshotApply::Stale,
} else { )
false | Err(_) => false,
} }
} }
} }
@@ -3013,6 +2989,7 @@ struct WorkerRecord {
worker_ref: WorkerRef, worker_ref: WorkerRef,
worker_id: WorkerId, worker_id: WorkerId,
status: WorkerStatus, status: WorkerStatus,
worker_state: Option<protocol::WorkerStateSnapshot>,
workspace_id: Option<String>, workspace_id: Option<String>,
request: CreateWorkerRequest, request: CreateWorkerRequest,
run_generation: u64, run_generation: u64,
@@ -3024,6 +3001,19 @@ struct WorkerRecord {
} }
impl WorkerRecord { impl WorkerRecord {
fn apply_worker_state(
&mut self,
incoming: &protocol::WorkerStateSnapshot,
) -> Result<protocol::WorkerStateSnapshotApply, protocol::WorkerStateSnapshotConflict> {
match self.worker_state.as_mut() {
Some(current) => protocol::apply_worker_state_snapshot(current, incoming),
None => {
self.worker_state = Some(incoming.clone());
Ok(protocol::WorkerStateSnapshotApply::Applied)
}
}
}
fn belongs_to_workspace(&self, workspace_id: &str) -> bool { fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
self.workspace_id.as_deref() == Some(workspace_id) self.workspace_id.as_deref() == Some(workspace_id)
} }
@@ -3033,6 +3023,7 @@ impl WorkerRecord {
worker_ref: self.worker_ref.clone(), worker_ref: self.worker_ref.clone(),
worker_id: self.worker_id, worker_id: self.worker_id,
status: self.status, status: self.status,
worker_state: self.worker_state.clone(),
workspace_id: self.workspace_id.clone(), workspace_id: self.workspace_id.clone(),
working_directory: self.working_directory.clone(), working_directory: self.working_directory.clone(),
profile: self.request.profile.clone(), profile: self.request.profile.clone(),
@@ -3047,6 +3038,7 @@ impl WorkerRecord {
worker_ref: self.worker_ref.clone(), worker_ref: self.worker_ref.clone(),
worker_id: self.worker_id, worker_id: self.worker_id,
status: self.status, status: self.status,
worker_state: self.worker_state.clone(),
workspace_id: self.workspace_id.clone(), workspace_id: self.workspace_id.clone(),
working_directory: self.working_directory.clone(), working_directory: self.working_directory.clone(),
profile: self.request.profile.clone(), profile: self.request.profile.clone(),
@@ -4719,7 +4711,7 @@ mod tests {
} }
#[test] #[test]
fn create_worker_uses_started_submission_ack_for_initial_running_status() { fn create_worker_does_not_infer_state_from_started_submission_ack() {
let (runtime, backend) = runtime_and_backend(); let (runtime, backend) = runtime_and_backend();
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
WorkerExecutionOperation::Input, WorkerExecutionOperation::Input,
@@ -4732,7 +4724,69 @@ mod tests {
let detail = runtime.create_worker(request).unwrap(); let detail = runtime.create_worker(request).unwrap();
assert_eq!(detail.status, WorkerStatus::Running); assert_eq!(detail.status, WorkerStatus::Idle);
assert_eq!(detail.worker_state, None);
}
#[test]
fn runtime_applies_only_newer_worker_state_snapshots() {
let (runtime, _) = runtime_and_backend();
let detail = runtime
.create_worker(task_request("state ordering"))
.unwrap();
let running = protocol::WorkerStateSnapshot {
execution_generation: 7,
revision: 3,
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)),
last_command_id: 2,
};
assert!({
let mut state = runtime.lock().unwrap();
state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: running.clone(),
},
)
});
assert_eq!(
runtime
.worker_detail(&detail.worker_ref)
.unwrap()
.worker_state,
Some(running.clone())
);
assert!({
let mut state = runtime.lock().unwrap();
!state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
revision: 2,
state: protocol::WorkerState::Idle,
..running.clone()
},
},
)
});
assert!({
let mut state = runtime.lock().unwrap();
!state.project_protocol_event_to_worker_state(
&detail.worker_ref,
&protocol::Event::WorkerState {
snapshot: protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Idle,
..running.clone()
},
},
)
});
let after = runtime.worker_detail(&detail.worker_ref).unwrap();
assert_eq!(after.status, WorkerStatus::Idle);
assert_eq!(after.worker_state, Some(running));
} }
#[test] #[test]
@@ -5027,10 +5081,9 @@ mod tests {
assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]); assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
assert_eq!( let restored = runtime.worker_detail(&detail.worker_ref).unwrap();
runtime.worker_detail(&detail.worker_ref).unwrap().status, assert_eq!(restored.status, WorkerStatus::Idle);
WorkerStatus::Running assert_eq!(restored.worker_state, None);
);
} }
#[test] #[test]
+112 -26
View File
@@ -17,7 +17,7 @@ use crate::ipc::notify_buffer::NotifyBuffer;
use crate::ipc::server::SocketServer; use crate::ipc::server::SocketServer;
use crate::runtime::dir::RuntimeDir; use crate::runtime::dir::RuntimeDir;
use crate::segment_log_sink::SegmentLogSink; use crate::segment_log_sink::SegmentLogSink;
use crate::shared_state::WorkerSharedState; use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState};
use crate::shutdown_after_idle::{ use crate::shutdown_after_idle::{
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role, ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
take_shutdown_request_after_status, take_shutdown_request_after_status,
@@ -181,21 +181,40 @@ impl WorkerHandle {
} }
} }
fn command_admission_disposition(
admission: WorkerCommandAdmission,
) -> Result<(), WorkerCommandDisposition> {
match admission {
WorkerCommandAdmission::Accepted => Ok(()),
WorkerCommandAdmission::Retry | WorkerCommandAdmission::StaleCommandId => {
Err(WorkerCommandDisposition::StaleCommandId)
}
WorkerCommandAdmission::Conflict => Err(WorkerCommandDisposition::Conflict),
WorkerCommandAdmission::ExecutionGenerationMismatch => {
Err(WorkerCommandDisposition::StaleExecutionGeneration)
}
WorkerCommandAdmission::StateRevisionMismatch => {
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
}
}
}
fn validate_command( fn validate_command(
envelope: WorkerCommandEnvelope,
kind: WorkerCommandKind,
shared_state: &WorkerSharedState,
) -> Result<(), WorkerCommandDisposition> {
command_admission_disposition(shared_state.admit_command(envelope, kind, true))
}
fn validate_shutdown_command(
envelope: WorkerCommandEnvelope, envelope: WorkerCommandEnvelope,
shared_state: &WorkerSharedState, shared_state: &WorkerSharedState,
) -> Result<(), WorkerCommandDisposition> { ) -> Result<(), WorkerCommandDisposition> {
let snapshot = shared_state.snapshot(); match shared_state.admit_command(envelope, WorkerCommandKind::Shutdown, false) {
if envelope.expected_execution_generation != snapshot.execution_generation { WorkerCommandAdmission::Accepted | WorkerCommandAdmission::Retry => Ok(()),
return Err(WorkerCommandDisposition::StaleExecutionGeneration); admission => command_admission_disposition(admission),
} }
if envelope.expected_worker_state_revision != snapshot.revision {
return Err(WorkerCommandDisposition::StaleWorkerStateRevision);
}
if !shared_state.accept_command_id(envelope.command_id) {
return Err(WorkerCommandDisposition::StaleCommandId);
}
Ok(())
} }
fn acknowledge_command( fn acknowledge_command(
@@ -205,6 +224,7 @@ fn acknowledge_command(
command: WorkerCommandKind, command: WorkerCommandKind,
disposition: WorkerCommandDisposition, disposition: WorkerCommandDisposition,
) { ) {
shared_state.complete_command(command_id, command, disposition);
let _ = working_event_tx.send(Event::CommandAcknowledged { let _ = working_event_tx.send(Event::CommandAcknowledged {
acknowledgement: WorkerCommandAcknowledgement { acknowledgement: WorkerCommandAcknowledgement {
command_id, command_id,
@@ -1913,7 +1933,9 @@ async fn controller_loop<C, St>(
} }
} }
Method::Resume { command } => { Method::Resume { command } => {
if let Err(disposition) = validate_command(command, &shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Resume, &shared_state)
{
acknowledge_command( acknowledge_command(
&working_event_tx, &working_event_tx,
&shared_state, &shared_state,
@@ -1953,7 +1975,9 @@ async fn controller_loop<C, St>(
} }
Method::Cancel { command } => { Method::Cancel { command } => {
if let Err(disposition) = validate_command(command, &shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Cancel, &shared_state)
{
acknowledge_command( acknowledge_command(
&working_event_tx, &working_event_tx,
&shared_state, &shared_state,
@@ -2017,7 +2041,9 @@ async fn controller_loop<C, St>(
} }
Method::Pause { command } => { Method::Pause { command } => {
if let Err(disposition) = validate_command(command, &shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Pause, &shared_state)
{
acknowledge_command( acknowledge_command(
&working_event_tx, &working_event_tx,
&shared_state, &shared_state,
@@ -2036,7 +2062,9 @@ async fn controller_loop<C, St>(
} }
Method::Compact { command } => { Method::Compact { command } => {
if let Err(disposition) = validate_command(command, &shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Compact, &shared_state)
{
acknowledge_command( acknowledge_command(
&working_event_tx, &working_event_tx,
&shared_state, &shared_state,
@@ -2081,7 +2109,11 @@ async fn controller_loop<C, St>(
method = method_rx.recv() => { method = method_rx.recv() => {
match method { match method {
Some(Method::Cancel { command }) => { Some(Method::Cancel { command }) => {
if let Err(disposition) = validate_command(command, &shared_state) { if let Err(disposition) = validate_command(
command,
WorkerCommandKind::Cancel,
&shared_state,
) {
acknowledge_command( acknowledge_command(
&working_event_tx, &working_event_tx,
&shared_state, &shared_state,
@@ -2101,7 +2133,18 @@ async fn controller_loop<C, St>(
let _ = cancel_tx.send(true); let _ = cancel_tx.send(true);
} }
Some(Method::Shutdown { command }) => { Some(Method::Shutdown { command }) => {
shared_state.accept_command_id(command.command_id); if let Err(disposition) =
validate_shutdown_command(command, &shared_state)
{
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
disposition,
);
continue;
}
shutdown_after_compaction = true; shutdown_after_compaction = true;
acknowledge_command( acknowledge_command(
&working_event_tx, &working_event_tx,
@@ -2196,9 +2239,18 @@ async fn controller_loop<C, St>(
}, },
Method::Shutdown { command } => { Method::Shutdown { command } => {
// Shutdown remains unconditional/retryable even when the caller's // Shutdown ignores the state-revision fence but remains bound to the
// live-state fence is stale. // current execution generation and command payload identity.
shared_state.accept_command_id(command.command_id); if let Err(disposition) = validate_shutdown_command(command, &shared_state) {
acknowledge_command(
&working_event_tx,
&shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
disposition,
);
continue;
}
acknowledge_command( acknowledge_command(
&working_event_tx, &working_event_tx,
&shared_state, &shared_state,
@@ -2544,7 +2596,9 @@ where
method = method_rx.recv(), if input_commit.is_none() => { method = method_rx.recv(), if input_commit.is_none() => {
match method { match method {
Some(Method::Cancel { command }) => { Some(Method::Cancel { command }) => {
if let Err(disposition) = validate_command(command, shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Cancel, shared_state)
{
acknowledge_command( acknowledge_command(
working_event_tx, working_event_tx,
shared_state, shared_state,
@@ -2583,7 +2637,9 @@ where
let _ = cancel_tx.try_send(()); let _ = cancel_tx.try_send(());
} }
Some(Method::Pause { command }) => { Some(Method::Pause { command }) => {
if let Err(disposition) = validate_command(command, shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Pause, shared_state)
{
acknowledge_command( acknowledge_command(
working_event_tx, working_event_tx,
shared_state, shared_state,
@@ -2623,7 +2679,16 @@ where
let _ = pause_tx.try_send(()); let _ = pause_tx.try_send(());
} }
Some(Method::Shutdown { command }) => { Some(Method::Shutdown { command }) => {
shared_state.accept_command_id(command.command_id); if let Err(disposition) = validate_shutdown_command(command, shared_state) {
acknowledge_command(
working_event_tx,
shared_state,
command.command_id,
WorkerCommandKind::Shutdown,
disposition,
);
continue;
}
shutdown_requested = true; shutdown_requested = true;
set_controller_state( set_controller_state(
shared_state, shared_state,
@@ -2705,7 +2770,9 @@ where
} }
} }
Some(Method::Resume { command }) => { Some(Method::Resume { command }) => {
if let Err(disposition) = validate_command(command, shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Resume, shared_state)
{
acknowledge_command( acknowledge_command(
working_event_tx, working_event_tx,
shared_state, shared_state,
@@ -2763,7 +2830,9 @@ where
} }
} }
Some(Method::Compact { command }) => { Some(Method::Compact { command }) => {
if let Err(disposition) = validate_command(command, shared_state) { if let Err(disposition) =
validate_command(command, WorkerCommandKind::Compact, shared_state)
{
acknowledge_command( acknowledge_command(
working_event_tx, working_event_tx,
shared_state, shared_state,
@@ -3677,6 +3746,7 @@ mod tests {
expected_execution_generation: 8, expected_execution_generation: 8,
expected_worker_state_revision: 0, expected_worker_state_revision: 0,
}, },
WorkerCommandKind::Pause,
&shared, &shared,
), ),
Err(WorkerCommandDisposition::StaleExecutionGeneration) Err(WorkerCommandDisposition::StaleExecutionGeneration)
@@ -3688,6 +3758,7 @@ mod tests {
expected_execution_generation: 9, expected_execution_generation: 9,
expected_worker_state_revision: 1, expected_worker_state_revision: 1,
}, },
WorkerCommandKind::Pause,
&shared, &shared,
), ),
Err(WorkerCommandDisposition::StaleWorkerStateRevision) Err(WorkerCommandDisposition::StaleWorkerStateRevision)
@@ -3699,6 +3770,7 @@ mod tests {
expected_execution_generation: 9, expected_execution_generation: 9,
expected_worker_state_revision: 0, expected_worker_state_revision: 0,
}, },
WorkerCommandKind::Pause,
&shared, &shared,
) )
.is_ok() .is_ok()
@@ -3708,12 +3780,25 @@ mod tests {
WorkerCommandEnvelope { WorkerCommandEnvelope {
command_id: 1, command_id: 1,
expected_execution_generation: 9, expected_execution_generation: 9,
expected_worker_state_revision: 1, expected_worker_state_revision: 0,
}, },
WorkerCommandKind::Pause,
&shared, &shared,
), ),
Err(WorkerCommandDisposition::StaleCommandId) Err(WorkerCommandDisposition::StaleCommandId)
); );
assert_eq!(
validate_command(
WorkerCommandEnvelope {
command_id: 1,
expected_execution_generation: 9,
expected_worker_state_revision: 0,
},
WorkerCommandKind::Cancel,
&shared,
),
Err(WorkerCommandDisposition::Conflict)
);
assert!( assert!(
validate_command( validate_command(
WorkerCommandEnvelope { WorkerCommandEnvelope {
@@ -3721,6 +3806,7 @@ mod tests {
expected_execution_generation: 9, expected_execution_generation: 9,
expected_worker_state_revision: 1, expected_worker_state_revision: 1,
}, },
WorkerCommandKind::Pause,
&shared, &shared,
) )
.is_ok() .is_ok()
+139 -10
View File
@@ -1,17 +1,37 @@
use std::collections::VecDeque;
use std::sync::{ use std::sync::{
OnceLock, RwLock, OnceLock, RwLock,
atomic::{AtomicBool, Ordering}, atomic::{AtomicBool, Ordering},
}; };
use protocol::{ use protocol::{
WorkerBusyState, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerBusyState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind,
WorkerStatus, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus,
}; };
use serde_json::json; use serde_json::json;
use session_store::SegmentId; use session_store::SegmentId;
use crate::fs_view::WorkerFsView; use crate::fs_view::WorkerFsView;
const COMPLETED_COMMAND_RETENTION: usize = 256;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct AcceptedWorkerCommand {
envelope: WorkerCommandEnvelope,
kind: WorkerCommandKind,
disposition: Option<WorkerCommandDisposition>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum WorkerCommandAdmission {
Accepted,
Retry,
Conflict,
StaleCommandId,
ExecutionGenerationMismatch,
StateRevisionMismatch,
}
/// Shared state between WorkerController and runtime directory. /// Shared state between WorkerController and runtime directory.
/// ///
/// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime /// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime
@@ -23,6 +43,7 @@ pub struct WorkerSharedState {
pub manifest_toml: String, pub manifest_toml: String,
pub greeting: protocol::Greeting, pub greeting: protocol::Greeting,
state: RwLock<WorkerStateSnapshot>, state: RwLock<WorkerStateSnapshot>,
accepted_commands: RwLock<VecDeque<AcceptedWorkerCommand>>,
/// Worker-from-the-inside view of the filesystem. Set once in /// Worker-from-the-inside view of the filesystem. Set once in
/// `WorkerController::start` after the local WorkdirSession provider is /// `WorkerController::start` after the local WorkdirSession provider is
/// materialised, and read from the IPC server layer to answer /// materialised, and read from the IPC server layer to answer
@@ -55,6 +76,7 @@ impl WorkerSharedState {
manifest_toml, manifest_toml,
greeting, greeting,
state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)), state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)),
accepted_commands: RwLock::new(VecDeque::new()),
fs_view: OnceLock::new(), fs_view: OnceLock::new(),
flow_transition_enabled: AtomicBool::new(false), flow_transition_enabled: AtomicBool::new(false),
} }
@@ -92,17 +114,99 @@ impl WorkerSharedState {
snapshot.clone() snapshot.clone()
} }
pub fn accept_command_id(&self, command_id: u64) -> bool { pub(crate) fn admit_command(
&self,
envelope: WorkerCommandEnvelope,
kind: WorkerCommandKind,
require_state_revision: bool,
) -> WorkerCommandAdmission {
let mut snapshot = self let mut snapshot = self
.state .state
.write() .write()
.expect("worker state lock poisoned; refusing command admission"); .expect("worker state lock poisoned; refusing command admission");
if command_id <= snapshot.last_command_id { let mut accepted = self
return false; .accepted_commands
.write()
.expect("worker command ledger lock poisoned; refusing command admission");
if let Some(existing) = accepted
.iter()
.find(|accepted| accepted.envelope.command_id == envelope.command_id)
{
return if existing.envelope == envelope && existing.kind == kind {
WorkerCommandAdmission::Retry
} else {
WorkerCommandAdmission::Conflict
};
} }
snapshot.last_command_id = command_id; if envelope.expected_execution_generation != snapshot.execution_generation {
return WorkerCommandAdmission::ExecutionGenerationMismatch;
}
if require_state_revision && envelope.expected_worker_state_revision != snapshot.revision {
return WorkerCommandAdmission::StateRevisionMismatch;
}
if envelope.command_id <= snapshot.last_command_id {
return WorkerCommandAdmission::StaleCommandId;
}
snapshot.last_command_id = envelope.command_id;
snapshot.revision = snapshot.revision.saturating_add(1); snapshot.revision = snapshot.revision.saturating_add(1);
true accepted.push_back(AcceptedWorkerCommand {
envelope,
kind,
disposition: None,
});
WorkerCommandAdmission::Accepted
}
pub(crate) fn complete_command(
&self,
command_id: u64,
kind: WorkerCommandKind,
disposition: WorkerCommandDisposition,
) {
if !matches!(
disposition,
WorkerCommandDisposition::Accepted | WorkerCommandDisposition::InvalidState
) {
return;
}
let mut accepted = self
.accepted_commands
.write()
.expect("worker command ledger lock poisoned; refusing command completion");
if let Some(command) = accepted
.iter_mut()
.find(|command| command.envelope.command_id == command_id && command.kind == kind)
{
command.disposition.get_or_insert(disposition);
}
while accepted
.iter()
.filter(|command| command.disposition.is_some())
.count()
> COMPLETED_COMMAND_RETENTION
{
let Some(index) = accepted
.iter()
.position(|command| command.disposition.is_some())
else {
break;
};
accepted.remove(index);
}
}
#[cfg(test)]
pub(crate) fn command_result(
&self,
command_id: u64,
) -> Option<Option<WorkerCommandDisposition>> {
self.accepted_commands
.read()
.expect("worker command ledger lock poisoned")
.iter()
.find(|command| command.envelope.command_id == command_id)
.map(|command| command.disposition)
} }
pub fn snapshot(&self) -> WorkerStateSnapshot { pub fn snapshot(&self) -> WorkerStateSnapshot {
@@ -190,9 +294,17 @@ mod tests {
} }
#[test] #[test]
fn accepted_command_id_advances_the_snapshot_revision_atomically() { fn accepted_command_identity_advances_revision_and_detects_reuse_conflicts() {
let state = test_state(); let state = test_state();
assert!(state.accept_command_id(9)); let envelope = WorkerCommandEnvelope {
command_id: 9,
expected_execution_generation: 7,
expected_worker_state_revision: 0,
};
assert_eq!(
state.admit_command(envelope, WorkerCommandKind::Pause, true),
WorkerCommandAdmission::Accepted
);
assert_eq!( assert_eq!(
state.snapshot(), state.snapshot(),
WorkerStateSnapshot { WorkerStateSnapshot {
@@ -202,7 +314,24 @@ mod tests {
state: WorkerState::Idle, state: WorkerState::Idle,
} }
); );
assert!(!state.accept_command_id(9)); assert_eq!(state.command_result(9), Some(None));
state.complete_command(
9,
WorkerCommandKind::Pause,
WorkerCommandDisposition::Accepted,
);
assert_eq!(
state.command_result(9),
Some(Some(WorkerCommandDisposition::Accepted))
);
assert_eq!(
state.admit_command(envelope, WorkerCommandKind::Pause, true),
WorkerCommandAdmission::Retry
);
assert_eq!(
state.admit_command(envelope, WorkerCommandKind::Cancel, true),
WorkerCommandAdmission::Conflict
);
assert_eq!(state.snapshot().revision, 1); assert_eq!(state.snapshot().revision, 1);
} }
+5
View File
@@ -1568,7 +1568,12 @@ pub struct WorkerSummary {
#[serde(default)] #[serde(default)]
pub tags: Vec<String>, pub tags: Vec<String>,
pub workspace: WorkerWorkspaceSummary, pub workspace: WorkerWorkspaceSummary,
/// Runtime catalog lifecycle compatibility state. Live foreground state, when
/// available, is carried separately in `worker_state`.
pub state: String, pub state: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "typescript", ts(optional))]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
pub last_seen_at: Option<String>, pub last_seen_at: Option<String>,
#[serde(default)] #[serde(default)]
pub pinned: bool, pub pinned: bool,
+10
View File
@@ -243,7 +243,10 @@ pub struct WorkerSummary {
#[serde(default)] #[serde(default)]
pub tags: Vec<String>, pub tags: Vec<String>,
pub workspace: WorkerWorkspaceSummary, pub workspace: WorkerWorkspaceSummary,
/// Runtime catalog lifecycle compatibility state.
pub state: String, pub state: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker_state: Option<protocol::WorkerStateSnapshot>,
pub last_seen_at: Option<String>, pub last_seen_at: Option<String>,
#[serde(default)] #[serde(default)]
pub pinned: bool, pub pinned: bool,
@@ -335,6 +338,7 @@ pub(crate) fn workspace_worker_summary(
workspace_id: summary.workspace.workspace_id, workspace_id: summary.workspace.workspace_id,
}, },
state: summary.state, state: summary.state,
worker_state: summary.worker_state,
last_seen_at: summary.last_seen_at, last_seen_at: summary.last_seen_at,
pinned: summary.pinned, pinned: summary.pinned,
retention_state: summary.retention_state, retention_state: summary.retention_state,
@@ -1998,6 +2002,7 @@ impl EmbeddedWorkerRuntime {
workspace_id: summary.workspace_id.clone(), workspace_id: summary.workspace_id.clone(),
}, },
state: embedded_worker_status_label(summary.status).to_string(), state: embedded_worker_status_label(summary.status).to_string(),
worker_state: summary.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -2037,6 +2042,7 @@ impl EmbeddedWorkerRuntime {
workspace_id: detail.workspace_id.clone(), workspace_id: detail.workspace_id.clone(),
}, },
state: embedded_worker_status_label(detail.status).to_string(), state: embedded_worker_status_label(detail.status).to_string(),
worker_state: detail.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -3340,6 +3346,7 @@ impl RemoteWorkerRuntime {
workspace_id: summary.workspace_id.clone(), workspace_id: summary.workspace_id.clone(),
}, },
state: embedded_worker_status_label(summary.status).to_string(), state: embedded_worker_status_label(summary.status).to_string(),
worker_state: summary.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -3383,6 +3390,7 @@ impl RemoteWorkerRuntime {
workspace_id: detail.workspace_id.clone(), workspace_id: detail.workspace_id.clone(),
}, },
state: embedded_worker_status_label(detail.status).to_string(), state: embedded_worker_status_label(detail.status).to_string(),
worker_state: detail.worker_state.clone(),
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -4730,6 +4738,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> WorkerSummary {
workspace_id: None, workspace_id: None,
}, },
state: "unsupported".to_string(), state: "unsupported".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -5251,6 +5260,7 @@ mod tests {
workspace_id: None, workspace_id: None,
}, },
state: "available".to_string(), state: "available".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "transient".to_string(), retention_state: "transient".to_string(),
@@ -202,7 +202,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
BrokerSubscriptionEvent::Event { BrokerSubscriptionEvent::Event {
payload: SubscriptionEventPayload::WorkerUpserted { ref worker }, payload: SubscriptionEventPayload::WorkerUpserted { ref worker },
.. ..
} if worker.state == SubscriptionWorkerState::Running } if worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running
)),
..
})
)
)); ));
} }
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap(); let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
@@ -212,7 +221,18 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
assert!(matches!( assert!(matches!(
snapshot, snapshot,
SubscriptionSnapshot::Workers { workers } SubscriptionSnapshot::Workers { workers }
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) if workers.iter().any(|worker| {
worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
),
..
})
)
})
)); ));
drop(late); drop(late);
@@ -337,8 +357,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
) )
.unwrap(); .unwrap();
assert!(matches!(next_event(&mut subscription).await, assert!(matches!(next_event(&mut subscription).await,
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. } BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running)); if worker.runtime_id.as_deref() == Some("embedded-worker-runtime")
&& worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running
)),
..
})
)));
let mut late = broker let mut late = broker
.subscribe( .subscribe(
"embedded-worker-runtime", "embedded-worker-runtime",
@@ -351,7 +381,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
assert!(matches!( assert!(matches!(
snapshot, snapshot,
SubscriptionSnapshot::Workers { workers } SubscriptionSnapshot::Workers { workers }
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) if workers.iter().any(|worker| {
worker.state == SubscriptionWorkerState::Idle
&& matches!(
worker.worker_state,
Some(protocol::WorkerStateSnapshot {
state: protocol::WorkerState::Busy(
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
),
..
})
)
})
)); ));
runtime runtime
+3
View File
@@ -15252,6 +15252,7 @@ fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> WorkerSummary
singleton_key: None, singleton_key: None,
tags: Vec::new(), tags: Vec::new(),
state: "missing".to_string(), state: "missing".to_string(),
worker_state: None,
last_seen_at: Some(record.updated_at.clone()), last_seen_at: Some(record.updated_at.clone()),
pinned: record.retention_state == "pinned", pinned: record.retention_state == "pinned",
retention_state: record.retention_state.clone(), retention_state: record.retention_state.clone(),
@@ -24990,6 +24991,7 @@ mod tests {
workspace_id: Some(TEST_WORKSPACE_ID.to_string()), workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
}, },
state: "idle".to_string(), state: "idle".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "normal".to_string(), retention_state: "normal".to_string(),
@@ -25086,6 +25088,7 @@ mod tests {
workspace_id: Some(TEST_WORKSPACE_ID.to_string()), workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
}, },
state: "idle".to_string(), state: "idle".to_string(),
worker_state: None,
last_seen_at: None, last_seen_at: None,
pinned: false, pinned: false,
retention_state: "normal".to_string(), retention_state: "normal".to_string(),
+11 -2
View File
@@ -19,7 +19,7 @@ command_id: number, expected_execution_generation: number, expected_worker_state
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown"; export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "invalid_state"; export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition, export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
/** /**
@@ -233,7 +233,16 @@ resource_key?: string | null,
/** /**
* Producer-owned monotonic revision for this Worker subject. * Producer-owned monotonic revision for this Worker subject.
*/ */
subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null, subject_revision: number,
/**
* Latest revisioned foreground state observed from the Worker. This remains
* absent until an authoritative Worker snapshot/event has been applied.
*/
worker_state?: WorkerStateSnapshot | null,
/**
* Runtime catalog lifecycle compatibility projection; not foreground-state authority.
*/
state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
/** /**
* Workspace-facing Repository key. Runtime producers leave this unset and * Workspace-facing Repository key. Runtime producers leave this unset and
* Workspace Server projections replace `repository_id` with this field. * Workspace Server projections replace `repository_id` with this field.
@@ -19,6 +19,7 @@ import type {
Event as PodProtocolEvent, Event as PodProtocolEvent,
Method as PodProtocolMethod, Method as PodProtocolMethod,
Segment as PodProtocolSegment, Segment as PodProtocolSegment,
WorkerStateSnapshot,
} from "$lib/generated/protocol"; } from "$lib/generated/protocol";
import type { import type {
GitCommitSummary as SharedGitCommitSummary, GitCommitSummary as SharedGitCommitSummary,
@@ -99,6 +100,7 @@ export type Worker = {
tags: string[]; tags: string[];
workspace: { visibility: string; identity: string }; workspace: { visibility: string; identity: string };
state: string; state: string;
worker_state?: WorkerStateSnapshot | null;
pinned?: boolean; pinned?: boolean;
retention_state?: string; retention_state?: string;
last_seen_at?: string | null; last_seen_at?: string | null;
@@ -0,0 +1,12 @@
import type { WorkerStateSnapshot } from "$lib/generated/protocol";
export function liveWorkerState(worker: {
state: string;
worker_state?: WorkerStateSnapshot | null;
}): string {
const state = worker.worker_state?.state;
if (!state) return worker.state === "stopped" ? "stopped" : "unknown";
if (state.kind === "idle") return "idle";
if (state.state.kind === "maintenance") return "running";
return state.state.state === "paused" ? "paused" : "running";
}
@@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void {
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
} }
} }
import { liveWorkerState } from './worker-state';
import { import {
applyWorkspaceWorkersFrame, applyWorkspaceWorkersFrame,
createWorkspaceWorkersProjection, createWorkspaceWorkersProjection,
@@ -33,6 +34,22 @@ function worker(
}; };
} }
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
const active = worker('runtime-a', 'worker-1', 1);
active.worker_state = {
execution_generation: 4,
revision: 2,
last_command_id: 1,
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
};
assertEquals(liveWorkerState(active), 'paused');
const unavailable = worker('runtime-a', 'worker-2', 1);
assertEquals(liveWorkerState(unavailable), 'unknown');
unavailable.state = 'stopped';
assertEquals(liveWorkerState(unavailable), 'stopped');
});
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => { Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
const projection = createWorkspaceWorkersProjection(); const projection = createWorkspaceWorkersProjection();
const frame: SubscriptionFrame = { const frame: SubscriptionFrame = {
@@ -5,6 +5,7 @@ import {
applyWorkspaceWorkersFrame, applyWorkspaceWorkersFrame,
createWorkspaceWorkersProjection, createWorkspaceWorkersProjection,
} from './worker-subscription-model'; } from './worker-subscription-model';
import { liveWorkerState } from './worker-state';
import { compareWorkersForSidebar } from './workers'; import { compareWorkersForSidebar } from './workers';
import type { Worker } from './types'; import type { Worker } from './types';
@@ -86,7 +87,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker {
profile: worker.profile ?? null, profile: worker.profile ?? null,
tags: [], tags: [],
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' }, workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
state: worker.state, state: liveWorkerState(worker),
worker_state: worker.worker_state,
pinned: false, pinned: false,
retention_state: 'transient', retention_state: 'transient',
implementation: { implementation: {
@@ -52,11 +52,7 @@
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store"; import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
import { workspaceApiPath } from "$lib/workspace/api/http"; import { workspaceApiPath } from "$lib/workspace/api/http";
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer"; import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
import type { import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types";
Diagnostic,
Worker,
PodProtocolEvent,
} from "$lib/workspace/sidebar/types";
type Props = { type Props = {
data: { data: {
@@ -207,7 +203,6 @@
); );
let pendingObservationEvents: ConsoleEventInput[] = []; let pendingObservationEvents: ConsoleEventInput[] = [];
let protocolEventSequence = 0; let protocolEventSequence = 0;
let pendingObservedStates: Array<string | null> = [];
let pendingStreamDiagnostics: Diagnostic[] = []; let pendingStreamDiagnostics: Diagnostic[] = [];
let observationFlushHandle: number | null = null; let observationFlushHandle: number | null = null;
let nextReloadToken = 0; let nextReloadToken = 0;
@@ -249,7 +244,9 @@
const diagnostics = $derived( const diagnostics = $derived(
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics), mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
); );
const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading"); const workerState = $derived(
liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"),
);
const workerRunning = $derived(workerState === "running"); const workerRunning = $derived(workerState === "running");
const workerPaused = $derived(workerState === "paused"); const workerPaused = $derived(workerState === "paused");
const composerEditable = $derived(protocolState === "open" && !sending); const composerEditable = $derived(protocolState === "open" && !sending);
@@ -343,7 +340,6 @@
observationFlushHandle = null; observationFlushHandle = null;
} }
pendingObservationEvents = []; pendingObservationEvents = [];
pendingObservedStates = [];
pendingStreamDiagnostics = []; pendingStreamDiagnostics = [];
} }
@@ -359,18 +355,15 @@
function flushObservationBatch() { function flushObservationBatch() {
observationFlushHandle = null; observationFlushHandle = null;
const eventBatch = pendingObservationEvents; const eventBatch = pendingObservationEvents;
const stateBatch = pendingObservedStates;
const diagnosticBatch = pendingStreamDiagnostics; const diagnosticBatch = pendingStreamDiagnostics;
pendingObservationEvents = []; pendingObservationEvents = [];
pendingObservedStates = [];
pendingStreamDiagnostics = []; pendingStreamDiagnostics = [];
if (eventBatch.length > 0) { if (eventBatch.length > 0) {
const latestState = stateBatch.findLast((state) => state !== null);
if (latestState) {
liveWorkerState = latestState;
}
consoleProjection = consoleProjector.append(eventBatch); consoleProjection = consoleProjector.append(eventBatch);
liveWorkerState = consoleProjection.status === "shutdown"
? "shutdown"
: workerStateFromSnapshot(consoleProjection.workerState);
advanceEventObservedAtVersion(); advanceEventObservedAtVersion();
} }
@@ -407,7 +400,6 @@
event: payload, event: payload,
observedAtMs, observedAtMs,
}); });
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
scheduleObservationFlush(); scheduleObservationFlush();
} }
@@ -925,36 +917,16 @@
handleComposerSubmit(); handleComposerSubmit();
} }
function workerStateFromProtocolEvent( function workerStateFromSnapshot(
event: PodProtocolEvent, snapshot: ConsoleProjection["workerState"],
): string | null { ): string | null {
switch (event.event) { if (!snapshot) return null;
case "snapshot": return snapshot.state.kind === "idle"
return event.data.state.state.kind === "idle" ? "idle"
? "idle" : snapshot.state.state.kind === "run" &&
: event.data.state.state.state.kind === "run" && snapshot.state.state.state === "paused"
event.data.state.state.state.state === "paused" ? "paused"
? "paused" : "running";
: "running";
case "worker_state":
return event.data.snapshot.state.kind === "idle"
? "idle"
: event.data.snapshot.state.state.kind === "run" &&
event.data.snapshot.state.state.state === "paused"
? "paused"
: "running";
case "command_acknowledged":
return event.data.acknowledgement.state.state.kind === "idle"
? "idle"
: event.data.acknowledgement.state.state.state.kind === "run" &&
event.data.acknowledgement.state.state.state.state === "paused"
? "paused"
: "running";
case "shutdown":
return "shutdown";
default:
return null;
}
} }
function connectProtocolTransport( function connectProtocolTransport(
@@ -4,6 +4,7 @@
import { workerHref } from '$lib/workspace/resource-links'; import { workerHref } from '$lib/workspace/resource-links';
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision'; import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers'; import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types'; import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
import type { PageProps } from './$types'; import type { PageProps } from './$types';
@@ -136,7 +137,7 @@
} }
function workerStatus(worker: Worker): string { function workerStatus(worker: Worker): string {
return worker.state; return liveWorkerState(worker);
} }
function workerProfile(worker: Worker): string { function workerProfile(worker: Worker): string {
@@ -1,5 +1,6 @@
<script lang="ts"> <script lang="ts">
import { workspaceRoute } from '$lib/workspace/api/http'; import { workspaceRoute } from '$lib/workspace/api/http';
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
import type { PageData } from './$types'; import type { PageData } from './$types';
let { data }: { data: PageData } = $props(); let { data }: { data: PageData } = $props();
</script> </script>
@@ -24,7 +25,7 @@
>Open console</a> >Open console</a>
</header> </header>
<dl class="resource-meta"> <dl class="resource-meta">
<dt>Status</dt><dd>{data.worker.state}</dd> <dt>Status</dt><dd>{liveWorkerState(data.worker)}</dd>
<dt>Profile</dt><dd>{data.worker.profile}</dd> <dt>Profile</dt><dd>{data.worker.profile}</dd>
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd> <dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
</dl> </dl>