fix: separate worker state from runtime lifecycle
This commit is contained in:
@@ -125,6 +125,7 @@ pub enum WorkerCommandDisposition {
|
|||||||
StaleExecutionGeneration,
|
StaleExecutionGeneration,
|
||||||
StaleWorkerStateRevision,
|
StaleWorkerStateRevision,
|
||||||
StaleCommandId,
|
StaleCommandId,
|
||||||
|
Conflict,
|
||||||
InvalidState,
|
InvalidState,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -573,6 +573,11 @@ pub struct SubscriptionWorker {
|
|||||||
pub resource_key: Option<String>,
|
pub resource_key: Option<String>,
|
||||||
/// Producer-owned monotonic revision for this Worker subject.
|
/// Producer-owned monotonic revision for this Worker subject.
|
||||||
pub subject_revision: u64,
|
pub subject_revision: u64,
|
||||||
|
/// Latest revisioned foreground state observed from the Worker. This remains
|
||||||
|
/// absent until an authoritative Worker snapshot/event has been applied.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<crate::WorkerStateSnapshot>,
|
||||||
|
/// Runtime catalog lifecycle compatibility projection; not foreground-state authority.
|
||||||
pub state: SubscriptionWorkerState,
|
pub state: SubscriptionWorkerState,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub has_running_internal_workers: bool,
|
pub has_running_internal_workers: bool,
|
||||||
@@ -874,6 +879,7 @@ mod tests {
|
|||||||
runtime_id: None,
|
runtime_id: None,
|
||||||
resource_key: None,
|
resource_key: None,
|
||||||
subject_revision: 0,
|
subject_revision: 0,
|
||||||
|
worker_state: None,
|
||||||
state: SubscriptionWorkerState::Idle,
|
state: SubscriptionWorkerState::Idle,
|
||||||
has_running_internal_workers: false,
|
has_running_internal_workers: false,
|
||||||
workspace_id: Some("workspace-1".to_string()),
|
workspace_id: Some("workspace-1".to_string()),
|
||||||
|
|||||||
+37
-16
@@ -1180,18 +1180,14 @@ impl App {
|
|||||||
self.assistant_streaming = false;
|
self.assistant_streaming = false;
|
||||||
}
|
}
|
||||||
Event::TurnStart { .. } => {
|
Event::TurnStart { .. } => {
|
||||||
self.set_worker_status(WorkerStatus::Running);
|
|
||||||
self.run_requests += 1;
|
self.run_requests += 1;
|
||||||
self.current_tool = None;
|
self.current_tool = None;
|
||||||
self.latest_llm_wait_event = None;
|
self.latest_llm_wait_event = None;
|
||||||
self.assistant_streaming = false;
|
self.assistant_streaming = false;
|
||||||
}
|
}
|
||||||
Event::InvokeStart { .. } => {
|
Event::InvokeStart { .. } => {}
|
||||||
self.set_worker_status(WorkerStatus::Running);
|
|
||||||
}
|
|
||||||
// UI consumers of per-attempt LlmCall semantics remain out of scope;
|
// UI consumers of per-attempt LlmCall semantics remain out of scope;
|
||||||
// the run-level status starts at InvokeStart and TurnStart counts each
|
// authoritative run state comes only from WorkerStateSnapshot.
|
||||||
// LLM request within that run.
|
|
||||||
Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => {
|
Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => {
|
||||||
self.latest_llm_wait_event = None;
|
self.latest_llm_wait_event = None;
|
||||||
}
|
}
|
||||||
@@ -1398,12 +1394,7 @@ impl App {
|
|||||||
output_tokens: self.run_output_tokens,
|
output_tokens: self.run_output_tokens,
|
||||||
});
|
});
|
||||||
self.pending_submit_rollback = None;
|
self.pending_submit_rollback = None;
|
||||||
self.reset_run_state(match result {
|
self.reset_run_state();
|
||||||
RunResult::Paused => WorkerStatus::Paused,
|
|
||||||
RunResult::Finished | RunResult::LimitReached | RunResult::RolledBack => {
|
|
||||||
WorkerStatus::Idle
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Event::CompactStart { .. } => {
|
Event::CompactStart { .. } => {
|
||||||
@@ -1536,7 +1527,7 @@ impl App {
|
|||||||
};
|
};
|
||||||
self.completion = None;
|
self.completion = None;
|
||||||
self.close_rewind_picker();
|
self.close_rewind_picker();
|
||||||
self.reset_run_state(self.worker_status);
|
self.reset_run_state();
|
||||||
let mut message = if restored_composer {
|
let mut message = if restored_composer {
|
||||||
format!(
|
format!(
|
||||||
"Rewound session: discarded {} log entries; restored selected input to composer.",
|
"Rewound session: discarded {} log entries; restored selected input to composer.",
|
||||||
@@ -1584,8 +1575,7 @@ impl App {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reset_run_state(&mut self, status: WorkerStatus) {
|
fn reset_run_state(&mut self) {
|
||||||
self.set_worker_status(status);
|
|
||||||
self.run_requests = 0;
|
self.run_requests = 0;
|
||||||
self.run_upload_tokens = 0;
|
self.run_upload_tokens = 0;
|
||||||
self.run_output_tokens = 0;
|
self.run_output_tokens = 0;
|
||||||
@@ -1615,7 +1605,7 @@ impl App {
|
|||||||
"Rolled back empty assistant turn; no local submitted input was available to restore."
|
"Rolled back empty assistant turn; no local submitted input was available to restore."
|
||||||
.to_owned()
|
.to_owned()
|
||||||
};
|
};
|
||||||
self.reset_run_state(WorkerStatus::Idle);
|
self.reset_run_state();
|
||||||
self.blocks.push(Block::Alert {
|
self.blocks.push(Block::Alert {
|
||||||
level: AlertLevel::Warn,
|
level: AlertLevel::Warn,
|
||||||
source: AlertSource::Worker,
|
source: AlertSource::Worker,
|
||||||
@@ -3583,6 +3573,37 @@ mod completion_flow_tests {
|
|||||||
assert!(matches!(app.blocks.first(), Some(Block::Greeting(_))));
|
assert!(matches!(app.blocks.first(), Some(Block::Greeting(_))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn occurrence_events_do_not_infer_foreground_worker_state() {
|
||||||
|
let mut app = App::new("test".into());
|
||||||
|
app.handle_worker_event(Event::TurnStart { turn: 1 });
|
||||||
|
app.handle_worker_event(Event::InvokeStart {
|
||||||
|
kind: protocol::InvokeKind::UserSend,
|
||||||
|
});
|
||||||
|
app.handle_worker_event(Event::RunEnd {
|
||||||
|
result: RunResult::Paused,
|
||||||
|
});
|
||||||
|
assert_eq!(app.worker_state.state, protocol::WorkerState::Idle);
|
||||||
|
assert_eq!(app.worker_status, WorkerStatus::Idle);
|
||||||
|
|
||||||
|
let running = WorkerStateSnapshot {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 1,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 0,
|
||||||
|
};
|
||||||
|
app.handle_worker_event(Event::WorkerState {
|
||||||
|
snapshot: running.clone(),
|
||||||
|
});
|
||||||
|
app.handle_worker_event(Event::RunEnd {
|
||||||
|
result: RunResult::Finished,
|
||||||
|
});
|
||||||
|
assert_eq!(app.worker_state, running);
|
||||||
|
assert_eq!(app.worker_status, WorkerStatus::Running);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn worker_state_events_and_acknowledgements_share_monotonic_application() {
|
fn worker_state_events_and_acknowledgements_share_monotonic_application() {
|
||||||
let mut app = App::new("test".into());
|
let mut app = App::new("test".into());
|
||||||
|
|||||||
@@ -321,7 +321,7 @@ fn row_line(
|
|||||||
Span::raw(" "),
|
Span::raw(" "),
|
||||||
Span::styled(
|
Span::styled(
|
||||||
pad_column(&worker_state(worker), widths.state),
|
pad_column(&worker_state(worker), widths.state),
|
||||||
state_style(worker.state.as_str()),
|
state_style(worker_state_label(worker)),
|
||||||
),
|
),
|
||||||
Span::raw(" "),
|
Span::raw(" "),
|
||||||
Span::styled(
|
Span::styled(
|
||||||
@@ -341,8 +341,20 @@ fn worker_name(worker: &BackendWorkerSummary) -> &str {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn worker_state_label(worker: &BackendWorkerSummary) -> &str {
|
||||||
|
match worker.worker_state.as_ref().map(|state| &state.state) {
|
||||||
|
Some(protocol::WorkerState::Idle) => "idle",
|
||||||
|
Some(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Paused,
|
||||||
|
))) => "paused",
|
||||||
|
Some(protocol::WorkerState::Busy(_)) => "running",
|
||||||
|
None if worker.state == "stopped" => "stopped",
|
||||||
|
None => "unknown",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn worker_state(worker: &BackendWorkerSummary) -> String {
|
fn worker_state(worker: &BackendWorkerSummary) -> String {
|
||||||
format!("[{}]", worker.state)
|
format!("[{}]", worker_state_label(worker))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn text_width(value: &str) -> usize {
|
fn text_width(value: &str) -> usize {
|
||||||
@@ -413,7 +425,15 @@ mod tests {
|
|||||||
identity: "ws".to_string(),
|
identity: "ws".to_string(),
|
||||||
workspace_id: Some("ws".to_string()),
|
workspace_id: Some("ws".to_string()),
|
||||||
},
|
},
|
||||||
state: "running".to_string(),
|
state: "idle".to_string(),
|
||||||
|
worker_state: Some(protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 1,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 0,
|
||||||
|
}),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: String::new(),
|
retention_state: String::new(),
|
||||||
@@ -450,6 +470,7 @@ mod tests {
|
|||||||
worker.display_name = "Coder".to_string();
|
worker.display_name = "Coder".to_string();
|
||||||
worker.label = "Coder · T-585".to_string();
|
worker.label = "Coder · T-585".to_string();
|
||||||
worker.state = "stopped".to_string();
|
worker.state = "stopped".to_string();
|
||||||
|
worker.worker_state = None;
|
||||||
worker.working_directory = Some(
|
worker.working_directory = Some(
|
||||||
serde_json::from_value(serde_json::json!({
|
serde_json::from_value(serde_json::json!({
|
||||||
"working_directory_id": "001a06a9f0202000000",
|
"working_directory_id": "001a06a9f0202000000",
|
||||||
@@ -478,12 +499,19 @@ mod tests {
|
|||||||
short.label = "Coder".to_string();
|
short.label = "Coder".to_string();
|
||||||
short.display_name = short.label.clone();
|
short.display_name = short.label.clone();
|
||||||
short.state = "idle".to_string();
|
short.state = "idle".to_string();
|
||||||
|
short.worker_state = Some(protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 1,
|
||||||
|
revision: 2,
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
last_command_id: 0,
|
||||||
|
});
|
||||||
|
|
||||||
let mut long = worker("runtime-a", "worker-b", None);
|
let mut long = worker("runtime-a", "worker-b", None);
|
||||||
long.resource_key = "W-100".to_string();
|
long.resource_key = "W-100".to_string();
|
||||||
long.label = "Longer worker · T-9".to_string();
|
long.label = "Longer worker · T-9".to_string();
|
||||||
long.display_name = long.label.clone();
|
long.display_name = long.label.clone();
|
||||||
long.state = "stopped".to_string();
|
long.state = "stopped".to_string();
|
||||||
|
long.worker_state = None;
|
||||||
|
|
||||||
for worker in [&mut short, &mut long] {
|
for worker in [&mut short, &mut long] {
|
||||||
worker.working_directory = Some(
|
worker.working_directory = Some(
|
||||||
|
|||||||
@@ -307,6 +307,8 @@ pub struct WorkerSummary {
|
|||||||
pub worker_id: WorkerId,
|
pub worker_id: WorkerId,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub workspace_id: Option<String>,
|
pub workspace_id: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||||
@@ -325,6 +327,8 @@ pub struct WorkerDetail {
|
|||||||
pub worker_id: WorkerId,
|
pub worker_id: WorkerId,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub workspace_id: Option<String>,
|
pub workspace_id: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub working_directory: Option<WorkingDirectoryStatus>,
|
pub working_directory: Option<WorkingDirectoryStatus>,
|
||||||
@@ -341,6 +345,8 @@ pub struct WorkerDetail {
|
|||||||
pub struct WorkerLifecycleAck {
|
pub struct WorkerLifecycleAck {
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub status: WorkerStatus,
|
pub status: WorkerStatus,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -3486,7 +3486,16 @@ mod ws_tests {
|
|||||||
..
|
..
|
||||||
}) if delivered_subscription_id == subscription_id
|
}) if delivered_subscription_id == subscription_id
|
||||||
&& worker.worker_id.as_str() == worker_ref.worker_id.to_string()
|
&& worker.worker_id.as_str() == worker_ref.worker_id.to_string()
|
||||||
&& worker.state == protocol::subscription::SubscriptionWorkerState::Running
|
&& worker.state == protocol::subscription::SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running
|
||||||
|
)),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
));
|
));
|
||||||
|
|
||||||
let unsubscribe_request_id =
|
let unsubscribe_request_id =
|
||||||
|
|||||||
@@ -700,6 +700,7 @@ impl Runtime {
|
|||||||
worker_ref: worker_ref.clone(),
|
worker_ref: worker_ref.clone(),
|
||||||
worker_id: worker_id.clone(),
|
worker_id: worker_id.clone(),
|
||||||
status: WorkerStatus::Stopped,
|
status: WorkerStatus::Stopped,
|
||||||
|
worker_state: None,
|
||||||
workspace_id: scope.map(|scope| scope.workspace_id.clone()),
|
workspace_id: scope.map(|scope| scope.workspace_id.clone()),
|
||||||
request: durable_request,
|
request: durable_request,
|
||||||
run_generation: 1,
|
run_generation: 1,
|
||||||
@@ -787,7 +788,6 @@ impl Runtime {
|
|||||||
let detail = self.commit_created_worker(
|
let detail = self.commit_created_worker(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
handle,
|
handle,
|
||||||
WorkerStatus::Running,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
dispatch_result,
|
dispatch_result,
|
||||||
)?;
|
)?;
|
||||||
@@ -797,7 +797,6 @@ impl Runtime {
|
|||||||
self.commit_created_worker(
|
self.commit_created_worker(
|
||||||
&worker_ref,
|
&worker_ref,
|
||||||
handle,
|
handle,
|
||||||
WorkerStatus::Idle,
|
|
||||||
working_directory,
|
working_directory,
|
||||||
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
|
WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn),
|
||||||
)
|
)
|
||||||
@@ -1220,17 +1219,7 @@ impl Runtime {
|
|||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
if let Some(snapshot) = dispatch_result.worker_state.as_ref() {
|
if let Some(snapshot) = dispatch_result.worker_state.as_ref() {
|
||||||
worker.status = match snapshot.catalog_status() {
|
let _ = worker.apply_worker_state(snapshot);
|
||||||
protocol::WorkerStatus::Idle => WorkerStatus::Idle,
|
|
||||||
protocol::WorkerStatus::Running => WorkerStatus::Running,
|
|
||||||
protocol::WorkerStatus::Paused => WorkerStatus::Paused,
|
|
||||||
protocol::WorkerStatus::Stopped => WorkerStatus::Stopped,
|
|
||||||
};
|
|
||||||
} else if matches!(
|
|
||||||
submission.as_ref().map(|ack| ack.disposition),
|
|
||||||
Some(protocol::SubmissionDisposition::Started)
|
|
||||||
) {
|
|
||||||
worker.status = WorkerStatus::Running;
|
|
||||||
}
|
}
|
||||||
let status = worker.status;
|
let status = worker.status;
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
@@ -1490,16 +1479,19 @@ impl Runtime {
|
|||||||
&self,
|
&self,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
handle: WorkerExecutionHandle,
|
handle: WorkerExecutionHandle,
|
||||||
status: WorkerStatus,
|
|
||||||
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
working_directory: Option<CatalogWorkingDirectoryStatus>,
|
||||||
_result: WorkerExecutionResult,
|
result: WorkerExecutionResult,
|
||||||
) -> Result<WorkerDetail, RuntimeError> {
|
) -> Result<WorkerDetail, RuntimeError> {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
let detail = {
|
let detail = {
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.execution_handle = Some(handle);
|
worker.execution_handle = Some(handle);
|
||||||
worker.execution_bound = true;
|
worker.execution_bound = true;
|
||||||
worker.status = status;
|
worker.status = WorkerStatus::Idle;
|
||||||
|
worker.worker_state = None;
|
||||||
|
if let Some(snapshot) = result.worker_state.as_ref() {
|
||||||
|
let _ = worker.apply_worker_state(snapshot);
|
||||||
|
}
|
||||||
worker.restore_intent = restore_intent_for_status(worker.status);
|
worker.restore_intent = restore_intent_for_status(worker.status);
|
||||||
worker.working_directory = working_directory;
|
worker.working_directory = working_directory;
|
||||||
worker.detail()
|
worker.detail()
|
||||||
@@ -1536,16 +1528,14 @@ impl Runtime {
|
|||||||
let Some(snapshot) = result.worker_state else {
|
let Some(snapshot) = result.worker_state else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
let status = match snapshot.catalog_status() {
|
|
||||||
protocol::WorkerStatus::Idle => WorkerStatus::Idle,
|
|
||||||
protocol::WorkerStatus::Running => WorkerStatus::Running,
|
|
||||||
protocol::WorkerStatus::Paused => WorkerStatus::Paused,
|
|
||||||
protocol::WorkerStatus::Stopped => WorkerStatus::Stopped,
|
|
||||||
};
|
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.status = status;
|
let applied = worker
|
||||||
worker.restore_intent = restore_intent_for_status(status);
|
.apply_worker_state(&snapshot)
|
||||||
|
.is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied));
|
||||||
|
if !applied {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||||
state.persist_runtime_snapshot()?;
|
state.persist_runtime_snapshot()?;
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
state.persist_worker(&worker_ref.worker_id)?;
|
||||||
@@ -1636,20 +1626,26 @@ impl Runtime {
|
|||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
reason: Option<String>,
|
reason: Option<String>,
|
||||||
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
||||||
let current = {
|
{
|
||||||
let state = self.lock()?;
|
let state = self.lock()?;
|
||||||
state.ensure_running()?;
|
state.ensure_running()?;
|
||||||
state.worker(worker_ref)?.status
|
if state.worker(worker_ref)?.status == WorkerStatus::Stopped {
|
||||||
};
|
return Ok(WorkerLifecycleAck {
|
||||||
if matches!(current, WorkerStatus::Idle | WorkerStatus::Stopped) {
|
worker_ref: worker_ref.clone(),
|
||||||
return Ok(WorkerLifecycleAck {
|
status: WorkerStatus::Stopped,
|
||||||
worker_ref: worker_ref.clone(),
|
worker_state: None,
|
||||||
status: current,
|
});
|
||||||
});
|
}
|
||||||
}
|
}
|
||||||
self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?;
|
self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?;
|
||||||
let _ = reason;
|
let _ = reason;
|
||||||
self.transition_worker_preserving_execution(worker_ref, WorkerStatus::Idle)
|
let state = self.lock()?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
Ok(WorkerLifecycleAck {
|
||||||
|
worker_ref: worker_ref.clone(),
|
||||||
|
status: worker.status,
|
||||||
|
worker_state: worker.worker_state.clone(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete a non-running Worker through a workspace-scoped Runtime authorization context.
|
/// Delete a non-running Worker through a workspace-scoped Runtime authorization context.
|
||||||
@@ -1795,12 +1791,13 @@ impl Runtime {
|
|||||||
) -> Result<WorkerObservationEvent, RuntimeError> {
|
) -> Result<WorkerObservationEvent, RuntimeError> {
|
||||||
let mut state = self.lock()?;
|
let mut state = self.lock()?;
|
||||||
state.ensure_worker_ref(worker_ref)?;
|
state.ensure_worker_ref(worker_ref)?;
|
||||||
let status_changed = state.project_protocol_event_to_status(worker_ref, &payload);
|
let worker_state_changed =
|
||||||
|
state.project_protocol_event_to_worker_state(worker_ref, &payload);
|
||||||
let activity_changed = state.project_internal_worker_activity(worker_ref, &payload);
|
let activity_changed = state.project_internal_worker_activity(worker_ref, &payload);
|
||||||
if status_changed || activity_changed {
|
if worker_state_changed || activity_changed {
|
||||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
state.publish_worker_upsert(worker_ref.worker_id)?;
|
||||||
}
|
}
|
||||||
if status_changed {
|
if worker_state_changed {
|
||||||
state.persist_runtime_snapshot()?;
|
state.persist_runtime_snapshot()?;
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
state.persist_worker(&worker_ref.worker_id)?;
|
||||||
}
|
}
|
||||||
@@ -1836,26 +1833,6 @@ impl Runtime {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn transition_worker_preserving_execution(
|
|
||||||
&self,
|
|
||||||
worker_ref: &WorkerRef,
|
|
||||||
status: WorkerStatus,
|
|
||||||
) -> Result<WorkerLifecycleAck, RuntimeError> {
|
|
||||||
let mut state = self.lock()?;
|
|
||||||
state.ensure_running()?;
|
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
|
||||||
worker.status = status;
|
|
||||||
worker.restore_intent = restore_intent_for_status(status);
|
|
||||||
let status = worker.status;
|
|
||||||
state.publish_worker_upsert(worker_ref.worker_id)?;
|
|
||||||
state.persist_runtime_snapshot()?;
|
|
||||||
state.persist_worker(&worker_ref.worker_id)?;
|
|
||||||
Ok(WorkerLifecycleAck {
|
|
||||||
worker_ref: worker_ref.clone(),
|
|
||||||
status,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn transition_worker(
|
fn transition_worker(
|
||||||
&self,
|
&self,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
@@ -1867,6 +1844,7 @@ impl Runtime {
|
|||||||
|
|
||||||
let worker = state.worker_mut(worker_ref)?;
|
let worker = state.worker_mut(worker_ref)?;
|
||||||
worker.status = status;
|
worker.status = status;
|
||||||
|
worker.worker_state = None;
|
||||||
worker.restore_intent = restore_intent_for_status(status);
|
worker.restore_intent = restore_intent_for_status(status);
|
||||||
worker.execution_handle = None;
|
worker.execution_handle = None;
|
||||||
worker.internal_workers.clear();
|
worker.internal_workers.clear();
|
||||||
@@ -1877,6 +1855,7 @@ impl Runtime {
|
|||||||
Ok(WorkerLifecycleAck {
|
Ok(WorkerLifecycleAck {
|
||||||
worker_ref: worker_ref.clone(),
|
worker_ref: worker_ref.clone(),
|
||||||
status,
|
status,
|
||||||
|
worker_state: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2301,6 +2280,7 @@ impl RuntimeState {
|
|||||||
worker_ref: worker.worker_ref,
|
worker_ref: worker.worker_ref,
|
||||||
worker_id: worker.worker_id,
|
worker_id: worker.worker_id,
|
||||||
status: worker.status,
|
status: worker.status,
|
||||||
|
worker_state: None,
|
||||||
workspace_id: worker.workspace_id,
|
workspace_id: worker.workspace_id,
|
||||||
request: worker.request,
|
request: worker.request,
|
||||||
run_generation,
|
run_generation,
|
||||||
@@ -2630,6 +2610,7 @@ impl RuntimeState {
|
|||||||
.get(&worker.worker_id)
|
.get(&worker.worker_id)
|
||||||
.copied()
|
.copied()
|
||||||
.unwrap_or(0),
|
.unwrap_or(0),
|
||||||
|
worker_state: worker.worker_state.clone(),
|
||||||
state: subscription_worker_state(worker.status),
|
state: subscription_worker_state(worker.status),
|
||||||
has_running_internal_workers: worker
|
has_running_internal_workers: worker
|
||||||
.internal_workers
|
.internal_workers
|
||||||
@@ -2965,7 +2946,7 @@ impl RuntimeState {
|
|||||||
Self::update_internal_worker_activity(&mut worker.internal_workers, event)
|
Self::update_internal_worker_activity(&mut worker.internal_workers, event)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn project_protocol_event_to_status(
|
fn project_protocol_event_to_worker_state(
|
||||||
&mut self,
|
&mut self,
|
||||||
worker_ref: &WorkerRef,
|
worker_ref: &WorkerRef,
|
||||||
event: &protocol::Event,
|
event: &protocol::Event,
|
||||||
@@ -2973,7 +2954,7 @@ impl RuntimeState {
|
|||||||
let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else {
|
let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let next_status = match event {
|
let incoming = match event {
|
||||||
protocol::Event::WorkerState { snapshot }
|
protocol::Event::WorkerState { snapshot }
|
||||||
| protocol::Event::Snapshot {
|
| protocol::Event::Snapshot {
|
||||||
state: snapshot, ..
|
state: snapshot, ..
|
||||||
@@ -2983,21 +2964,16 @@ impl RuntimeState {
|
|||||||
protocol::WorkerCommandAcknowledgement {
|
protocol::WorkerCommandAcknowledgement {
|
||||||
state: snapshot, ..
|
state: snapshot, ..
|
||||||
},
|
},
|
||||||
} => Some(match snapshot.catalog_status() {
|
} => snapshot,
|
||||||
protocol::WorkerStatus::Idle => WorkerStatus::Idle,
|
_ => return false,
|
||||||
protocol::WorkerStatus::Running => WorkerStatus::Running,
|
|
||||||
protocol::WorkerStatus::Paused => WorkerStatus::Paused,
|
|
||||||
protocol::WorkerStatus::Stopped => WorkerStatus::Stopped,
|
|
||||||
}),
|
|
||||||
_ => None,
|
|
||||||
};
|
};
|
||||||
if let Some(next_status) = next_status {
|
match worker.apply_worker_state(incoming) {
|
||||||
let changed = worker.status != next_status;
|
Ok(protocol::WorkerStateSnapshotApply::Applied) => true,
|
||||||
worker.status = next_status;
|
Ok(
|
||||||
worker.restore_intent = restore_intent_for_status(next_status);
|
protocol::WorkerStateSnapshotApply::Duplicate
|
||||||
changed
|
| protocol::WorkerStateSnapshotApply::Stale,
|
||||||
} else {
|
)
|
||||||
false
|
| Err(_) => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3013,6 +2989,7 @@ struct WorkerRecord {
|
|||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
worker_id: WorkerId,
|
worker_id: WorkerId,
|
||||||
status: WorkerStatus,
|
status: WorkerStatus,
|
||||||
|
worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
workspace_id: Option<String>,
|
workspace_id: Option<String>,
|
||||||
request: CreateWorkerRequest,
|
request: CreateWorkerRequest,
|
||||||
run_generation: u64,
|
run_generation: u64,
|
||||||
@@ -3024,6 +3001,19 @@ struct WorkerRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerRecord {
|
impl WorkerRecord {
|
||||||
|
fn apply_worker_state(
|
||||||
|
&mut self,
|
||||||
|
incoming: &protocol::WorkerStateSnapshot,
|
||||||
|
) -> Result<protocol::WorkerStateSnapshotApply, protocol::WorkerStateSnapshotConflict> {
|
||||||
|
match self.worker_state.as_mut() {
|
||||||
|
Some(current) => protocol::apply_worker_state_snapshot(current, incoming),
|
||||||
|
None => {
|
||||||
|
self.worker_state = Some(incoming.clone());
|
||||||
|
Ok(protocol::WorkerStateSnapshotApply::Applied)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
|
fn belongs_to_workspace(&self, workspace_id: &str) -> bool {
|
||||||
self.workspace_id.as_deref() == Some(workspace_id)
|
self.workspace_id.as_deref() == Some(workspace_id)
|
||||||
}
|
}
|
||||||
@@ -3033,6 +3023,7 @@ impl WorkerRecord {
|
|||||||
worker_ref: self.worker_ref.clone(),
|
worker_ref: self.worker_ref.clone(),
|
||||||
worker_id: self.worker_id,
|
worker_id: self.worker_id,
|
||||||
status: self.status,
|
status: self.status,
|
||||||
|
worker_state: self.worker_state.clone(),
|
||||||
workspace_id: self.workspace_id.clone(),
|
workspace_id: self.workspace_id.clone(),
|
||||||
working_directory: self.working_directory.clone(),
|
working_directory: self.working_directory.clone(),
|
||||||
profile: self.request.profile.clone(),
|
profile: self.request.profile.clone(),
|
||||||
@@ -3047,6 +3038,7 @@ impl WorkerRecord {
|
|||||||
worker_ref: self.worker_ref.clone(),
|
worker_ref: self.worker_ref.clone(),
|
||||||
worker_id: self.worker_id,
|
worker_id: self.worker_id,
|
||||||
status: self.status,
|
status: self.status,
|
||||||
|
worker_state: self.worker_state.clone(),
|
||||||
workspace_id: self.workspace_id.clone(),
|
workspace_id: self.workspace_id.clone(),
|
||||||
working_directory: self.working_directory.clone(),
|
working_directory: self.working_directory.clone(),
|
||||||
profile: self.request.profile.clone(),
|
profile: self.request.profile.clone(),
|
||||||
@@ -4719,7 +4711,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn create_worker_uses_started_submission_ack_for_initial_running_status() {
|
fn create_worker_does_not_infer_state_from_started_submission_ack() {
|
||||||
let (runtime, backend) = runtime_and_backend();
|
let (runtime, backend) = runtime_and_backend();
|
||||||
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
backend.set_dispatch_result(WorkerExecutionResult::accepted_submission(
|
||||||
WorkerExecutionOperation::Input,
|
WorkerExecutionOperation::Input,
|
||||||
@@ -4732,7 +4724,69 @@ mod tests {
|
|||||||
|
|
||||||
let detail = runtime.create_worker(request).unwrap();
|
let detail = runtime.create_worker(request).unwrap();
|
||||||
|
|
||||||
assert_eq!(detail.status, WorkerStatus::Running);
|
assert_eq!(detail.status, WorkerStatus::Idle);
|
||||||
|
assert_eq!(detail.worker_state, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_applies_only_newer_worker_state_snapshots() {
|
||||||
|
let (runtime, _) = runtime_and_backend();
|
||||||
|
let detail = runtime
|
||||||
|
.create_worker(task_request("state ordering"))
|
||||||
|
.unwrap();
|
||||||
|
let running = protocol::WorkerStateSnapshot {
|
||||||
|
execution_generation: 7,
|
||||||
|
revision: 3,
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running,
|
||||||
|
)),
|
||||||
|
last_command_id: 2,
|
||||||
|
};
|
||||||
|
assert!({
|
||||||
|
let mut state = runtime.lock().unwrap();
|
||||||
|
state.project_protocol_event_to_worker_state(
|
||||||
|
&detail.worker_ref,
|
||||||
|
&protocol::Event::WorkerState {
|
||||||
|
snapshot: running.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&detail.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.worker_state,
|
||||||
|
Some(running.clone())
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!({
|
||||||
|
let mut state = runtime.lock().unwrap();
|
||||||
|
!state.project_protocol_event_to_worker_state(
|
||||||
|
&detail.worker_ref,
|
||||||
|
&protocol::Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
revision: 2,
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..running.clone()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
});
|
||||||
|
assert!({
|
||||||
|
let mut state = runtime.lock().unwrap();
|
||||||
|
!state.project_protocol_event_to_worker_state(
|
||||||
|
&detail.worker_ref,
|
||||||
|
&protocol::Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..running.clone()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
});
|
||||||
|
let after = runtime.worker_detail(&detail.worker_ref).unwrap();
|
||||||
|
assert_eq!(after.status, WorkerStatus::Idle);
|
||||||
|
assert_eq!(after.worker_state, Some(running));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -5027,10 +5081,9 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
|
assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]);
|
||||||
assert_eq!(
|
let restored = runtime.worker_detail(&detail.worker_ref).unwrap();
|
||||||
runtime.worker_detail(&detail.worker_ref).unwrap().status,
|
assert_eq!(restored.status, WorkerStatus::Idle);
|
||||||
WorkerStatus::Running
|
assert_eq!(restored.worker_state, None);
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+112
-26
@@ -17,7 +17,7 @@ use crate::ipc::notify_buffer::NotifyBuffer;
|
|||||||
use crate::ipc::server::SocketServer;
|
use crate::ipc::server::SocketServer;
|
||||||
use crate::runtime::dir::RuntimeDir;
|
use crate::runtime::dir::RuntimeDir;
|
||||||
use crate::segment_log_sink::SegmentLogSink;
|
use crate::segment_log_sink::SegmentLogSink;
|
||||||
use crate::shared_state::WorkerSharedState;
|
use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState};
|
||||||
use crate::shutdown_after_idle::{
|
use crate::shutdown_after_idle::{
|
||||||
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
||||||
take_shutdown_request_after_status,
|
take_shutdown_request_after_status,
|
||||||
@@ -181,21 +181,40 @@ impl WorkerHandle {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn command_admission_disposition(
|
||||||
|
admission: WorkerCommandAdmission,
|
||||||
|
) -> Result<(), WorkerCommandDisposition> {
|
||||||
|
match admission {
|
||||||
|
WorkerCommandAdmission::Accepted => Ok(()),
|
||||||
|
WorkerCommandAdmission::Retry | WorkerCommandAdmission::StaleCommandId => {
|
||||||
|
Err(WorkerCommandDisposition::StaleCommandId)
|
||||||
|
}
|
||||||
|
WorkerCommandAdmission::Conflict => Err(WorkerCommandDisposition::Conflict),
|
||||||
|
WorkerCommandAdmission::ExecutionGenerationMismatch => {
|
||||||
|
Err(WorkerCommandDisposition::StaleExecutionGeneration)
|
||||||
|
}
|
||||||
|
WorkerCommandAdmission::StateRevisionMismatch => {
|
||||||
|
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_command(
|
fn validate_command(
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
shared_state: &WorkerSharedState,
|
||||||
|
) -> Result<(), WorkerCommandDisposition> {
|
||||||
|
command_admission_disposition(shared_state.admit_command(envelope, kind, true))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_shutdown_command(
|
||||||
envelope: WorkerCommandEnvelope,
|
envelope: WorkerCommandEnvelope,
|
||||||
shared_state: &WorkerSharedState,
|
shared_state: &WorkerSharedState,
|
||||||
) -> Result<(), WorkerCommandDisposition> {
|
) -> Result<(), WorkerCommandDisposition> {
|
||||||
let snapshot = shared_state.snapshot();
|
match shared_state.admit_command(envelope, WorkerCommandKind::Shutdown, false) {
|
||||||
if envelope.expected_execution_generation != snapshot.execution_generation {
|
WorkerCommandAdmission::Accepted | WorkerCommandAdmission::Retry => Ok(()),
|
||||||
return Err(WorkerCommandDisposition::StaleExecutionGeneration);
|
admission => command_admission_disposition(admission),
|
||||||
}
|
}
|
||||||
if envelope.expected_worker_state_revision != snapshot.revision {
|
|
||||||
return Err(WorkerCommandDisposition::StaleWorkerStateRevision);
|
|
||||||
}
|
|
||||||
if !shared_state.accept_command_id(envelope.command_id) {
|
|
||||||
return Err(WorkerCommandDisposition::StaleCommandId);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn acknowledge_command(
|
fn acknowledge_command(
|
||||||
@@ -205,6 +224,7 @@ fn acknowledge_command(
|
|||||||
command: WorkerCommandKind,
|
command: WorkerCommandKind,
|
||||||
disposition: WorkerCommandDisposition,
|
disposition: WorkerCommandDisposition,
|
||||||
) {
|
) {
|
||||||
|
shared_state.complete_command(command_id, command, disposition);
|
||||||
let _ = working_event_tx.send(Event::CommandAcknowledged {
|
let _ = working_event_tx.send(Event::CommandAcknowledged {
|
||||||
acknowledgement: WorkerCommandAcknowledgement {
|
acknowledgement: WorkerCommandAcknowledgement {
|
||||||
command_id,
|
command_id,
|
||||||
@@ -1913,7 +1933,9 @@ async fn controller_loop<C, St>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Method::Resume { command } => {
|
Method::Resume { command } => {
|
||||||
if let Err(disposition) = validate_command(command, &shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Resume, &shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -1953,7 +1975,9 @@ async fn controller_loop<C, St>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Method::Cancel { command } => {
|
Method::Cancel { command } => {
|
||||||
if let Err(disposition) = validate_command(command, &shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Cancel, &shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -2017,7 +2041,9 @@ async fn controller_loop<C, St>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Method::Pause { command } => {
|
Method::Pause { command } => {
|
||||||
if let Err(disposition) = validate_command(command, &shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Pause, &shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -2036,7 +2062,9 @@ async fn controller_loop<C, St>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Method::Compact { command } => {
|
Method::Compact { command } => {
|
||||||
if let Err(disposition) = validate_command(command, &shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Compact, &shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -2081,7 +2109,11 @@ async fn controller_loop<C, St>(
|
|||||||
method = method_rx.recv() => {
|
method = method_rx.recv() => {
|
||||||
match method {
|
match method {
|
||||||
Some(Method::Cancel { command }) => {
|
Some(Method::Cancel { command }) => {
|
||||||
if let Err(disposition) = validate_command(command, &shared_state) {
|
if let Err(disposition) = validate_command(
|
||||||
|
command,
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
&shared_state,
|
||||||
|
) {
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -2101,7 +2133,18 @@ async fn controller_loop<C, St>(
|
|||||||
let _ = cancel_tx.send(true);
|
let _ = cancel_tx.send(true);
|
||||||
}
|
}
|
||||||
Some(Method::Shutdown { command }) => {
|
Some(Method::Shutdown { command }) => {
|
||||||
shared_state.accept_command_id(command.command_id);
|
if let Err(disposition) =
|
||||||
|
validate_shutdown_command(command, &shared_state)
|
||||||
|
{
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
shutdown_after_compaction = true;
|
shutdown_after_compaction = true;
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
@@ -2196,9 +2239,18 @@ async fn controller_loop<C, St>(
|
|||||||
},
|
},
|
||||||
|
|
||||||
Method::Shutdown { command } => {
|
Method::Shutdown { command } => {
|
||||||
// Shutdown remains unconditional/retryable even when the caller's
|
// Shutdown ignores the state-revision fence but remains bound to the
|
||||||
// live-state fence is stale.
|
// current execution generation and command payload identity.
|
||||||
shared_state.accept_command_id(command.command_id);
|
if let Err(disposition) = validate_shutdown_command(command, &shared_state) {
|
||||||
|
acknowledge_command(
|
||||||
|
&working_event_tx,
|
||||||
|
&shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
&working_event_tx,
|
&working_event_tx,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
@@ -2544,7 +2596,9 @@ where
|
|||||||
method = method_rx.recv(), if input_commit.is_none() => {
|
method = method_rx.recv(), if input_commit.is_none() => {
|
||||||
match method {
|
match method {
|
||||||
Some(Method::Cancel { command }) => {
|
Some(Method::Cancel { command }) => {
|
||||||
if let Err(disposition) = validate_command(command, shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Cancel, shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
working_event_tx,
|
working_event_tx,
|
||||||
shared_state,
|
shared_state,
|
||||||
@@ -2583,7 +2637,9 @@ where
|
|||||||
let _ = cancel_tx.try_send(());
|
let _ = cancel_tx.try_send(());
|
||||||
}
|
}
|
||||||
Some(Method::Pause { command }) => {
|
Some(Method::Pause { command }) => {
|
||||||
if let Err(disposition) = validate_command(command, shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Pause, shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
working_event_tx,
|
working_event_tx,
|
||||||
shared_state,
|
shared_state,
|
||||||
@@ -2623,7 +2679,16 @@ where
|
|||||||
let _ = pause_tx.try_send(());
|
let _ = pause_tx.try_send(());
|
||||||
}
|
}
|
||||||
Some(Method::Shutdown { command }) => {
|
Some(Method::Shutdown { command }) => {
|
||||||
shared_state.accept_command_id(command.command_id);
|
if let Err(disposition) = validate_shutdown_command(command, shared_state) {
|
||||||
|
acknowledge_command(
|
||||||
|
working_event_tx,
|
||||||
|
shared_state,
|
||||||
|
command.command_id,
|
||||||
|
WorkerCommandKind::Shutdown,
|
||||||
|
disposition,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
shutdown_requested = true;
|
shutdown_requested = true;
|
||||||
set_controller_state(
|
set_controller_state(
|
||||||
shared_state,
|
shared_state,
|
||||||
@@ -2705,7 +2770,9 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some(Method::Resume { command }) => {
|
Some(Method::Resume { command }) => {
|
||||||
if let Err(disposition) = validate_command(command, shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Resume, shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
working_event_tx,
|
working_event_tx,
|
||||||
shared_state,
|
shared_state,
|
||||||
@@ -2763,7 +2830,9 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some(Method::Compact { command }) => {
|
Some(Method::Compact { command }) => {
|
||||||
if let Err(disposition) = validate_command(command, shared_state) {
|
if let Err(disposition) =
|
||||||
|
validate_command(command, WorkerCommandKind::Compact, shared_state)
|
||||||
|
{
|
||||||
acknowledge_command(
|
acknowledge_command(
|
||||||
working_event_tx,
|
working_event_tx,
|
||||||
shared_state,
|
shared_state,
|
||||||
@@ -3677,6 +3746,7 @@ mod tests {
|
|||||||
expected_execution_generation: 8,
|
expected_execution_generation: 8,
|
||||||
expected_worker_state_revision: 0,
|
expected_worker_state_revision: 0,
|
||||||
},
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
&shared,
|
&shared,
|
||||||
),
|
),
|
||||||
Err(WorkerCommandDisposition::StaleExecutionGeneration)
|
Err(WorkerCommandDisposition::StaleExecutionGeneration)
|
||||||
@@ -3688,6 +3758,7 @@ mod tests {
|
|||||||
expected_execution_generation: 9,
|
expected_execution_generation: 9,
|
||||||
expected_worker_state_revision: 1,
|
expected_worker_state_revision: 1,
|
||||||
},
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
&shared,
|
&shared,
|
||||||
),
|
),
|
||||||
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
|
Err(WorkerCommandDisposition::StaleWorkerStateRevision)
|
||||||
@@ -3699,6 +3770,7 @@ mod tests {
|
|||||||
expected_execution_generation: 9,
|
expected_execution_generation: 9,
|
||||||
expected_worker_state_revision: 0,
|
expected_worker_state_revision: 0,
|
||||||
},
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
&shared,
|
&shared,
|
||||||
)
|
)
|
||||||
.is_ok()
|
.is_ok()
|
||||||
@@ -3708,12 +3780,25 @@ mod tests {
|
|||||||
WorkerCommandEnvelope {
|
WorkerCommandEnvelope {
|
||||||
command_id: 1,
|
command_id: 1,
|
||||||
expected_execution_generation: 9,
|
expected_execution_generation: 9,
|
||||||
expected_worker_state_revision: 1,
|
expected_worker_state_revision: 0,
|
||||||
},
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
&shared,
|
&shared,
|
||||||
),
|
),
|
||||||
Err(WorkerCommandDisposition::StaleCommandId)
|
Err(WorkerCommandDisposition::StaleCommandId)
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
validate_command(
|
||||||
|
WorkerCommandEnvelope {
|
||||||
|
command_id: 1,
|
||||||
|
expected_execution_generation: 9,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
},
|
||||||
|
WorkerCommandKind::Cancel,
|
||||||
|
&shared,
|
||||||
|
),
|
||||||
|
Err(WorkerCommandDisposition::Conflict)
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
validate_command(
|
validate_command(
|
||||||
WorkerCommandEnvelope {
|
WorkerCommandEnvelope {
|
||||||
@@ -3721,6 +3806,7 @@ mod tests {
|
|||||||
expected_execution_generation: 9,
|
expected_execution_generation: 9,
|
||||||
expected_worker_state_revision: 1,
|
expected_worker_state_revision: 1,
|
||||||
},
|
},
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
&shared,
|
&shared,
|
||||||
)
|
)
|
||||||
.is_ok()
|
.is_ok()
|
||||||
|
|||||||
@@ -1,17 +1,37 @@
|
|||||||
|
use std::collections::VecDeque;
|
||||||
use std::sync::{
|
use std::sync::{
|
||||||
OnceLock, RwLock,
|
OnceLock, RwLock,
|
||||||
atomic::{AtomicBool, Ordering},
|
atomic::{AtomicBool, Ordering},
|
||||||
};
|
};
|
||||||
|
|
||||||
use protocol::{
|
use protocol::{
|
||||||
WorkerBusyState, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot,
|
WorkerBusyState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind,
|
||||||
WorkerStatus,
|
WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus,
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use session_store::SegmentId;
|
use session_store::SegmentId;
|
||||||
|
|
||||||
use crate::fs_view::WorkerFsView;
|
use crate::fs_view::WorkerFsView;
|
||||||
|
|
||||||
|
const COMPLETED_COMMAND_RETENTION: usize = 256;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
struct AcceptedWorkerCommand {
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
disposition: Option<WorkerCommandDisposition>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum WorkerCommandAdmission {
|
||||||
|
Accepted,
|
||||||
|
Retry,
|
||||||
|
Conflict,
|
||||||
|
StaleCommandId,
|
||||||
|
ExecutionGenerationMismatch,
|
||||||
|
StateRevisionMismatch,
|
||||||
|
}
|
||||||
|
|
||||||
/// Shared state between WorkerController and runtime directory.
|
/// Shared state between WorkerController and runtime directory.
|
||||||
///
|
///
|
||||||
/// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime
|
/// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime
|
||||||
@@ -23,6 +43,7 @@ pub struct WorkerSharedState {
|
|||||||
pub manifest_toml: String,
|
pub manifest_toml: String,
|
||||||
pub greeting: protocol::Greeting,
|
pub greeting: protocol::Greeting,
|
||||||
state: RwLock<WorkerStateSnapshot>,
|
state: RwLock<WorkerStateSnapshot>,
|
||||||
|
accepted_commands: RwLock<VecDeque<AcceptedWorkerCommand>>,
|
||||||
/// Worker-from-the-inside view of the filesystem. Set once in
|
/// Worker-from-the-inside view of the filesystem. Set once in
|
||||||
/// `WorkerController::start` after the local WorkdirSession provider is
|
/// `WorkerController::start` after the local WorkdirSession provider is
|
||||||
/// materialised, and read from the IPC server layer to answer
|
/// materialised, and read from the IPC server layer to answer
|
||||||
@@ -55,6 +76,7 @@ impl WorkerSharedState {
|
|||||||
manifest_toml,
|
manifest_toml,
|
||||||
greeting,
|
greeting,
|
||||||
state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)),
|
state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)),
|
||||||
|
accepted_commands: RwLock::new(VecDeque::new()),
|
||||||
fs_view: OnceLock::new(),
|
fs_view: OnceLock::new(),
|
||||||
flow_transition_enabled: AtomicBool::new(false),
|
flow_transition_enabled: AtomicBool::new(false),
|
||||||
}
|
}
|
||||||
@@ -92,17 +114,99 @@ impl WorkerSharedState {
|
|||||||
snapshot.clone()
|
snapshot.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn accept_command_id(&self, command_id: u64) -> bool {
|
pub(crate) fn admit_command(
|
||||||
|
&self,
|
||||||
|
envelope: WorkerCommandEnvelope,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
require_state_revision: bool,
|
||||||
|
) -> WorkerCommandAdmission {
|
||||||
let mut snapshot = self
|
let mut snapshot = self
|
||||||
.state
|
.state
|
||||||
.write()
|
.write()
|
||||||
.expect("worker state lock poisoned; refusing command admission");
|
.expect("worker state lock poisoned; refusing command admission");
|
||||||
if command_id <= snapshot.last_command_id {
|
let mut accepted = self
|
||||||
return false;
|
.accepted_commands
|
||||||
|
.write()
|
||||||
|
.expect("worker command ledger lock poisoned; refusing command admission");
|
||||||
|
if let Some(existing) = accepted
|
||||||
|
.iter()
|
||||||
|
.find(|accepted| accepted.envelope.command_id == envelope.command_id)
|
||||||
|
{
|
||||||
|
return if existing.envelope == envelope && existing.kind == kind {
|
||||||
|
WorkerCommandAdmission::Retry
|
||||||
|
} else {
|
||||||
|
WorkerCommandAdmission::Conflict
|
||||||
|
};
|
||||||
}
|
}
|
||||||
snapshot.last_command_id = command_id;
|
if envelope.expected_execution_generation != snapshot.execution_generation {
|
||||||
|
return WorkerCommandAdmission::ExecutionGenerationMismatch;
|
||||||
|
}
|
||||||
|
if require_state_revision && envelope.expected_worker_state_revision != snapshot.revision {
|
||||||
|
return WorkerCommandAdmission::StateRevisionMismatch;
|
||||||
|
}
|
||||||
|
if envelope.command_id <= snapshot.last_command_id {
|
||||||
|
return WorkerCommandAdmission::StaleCommandId;
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot.last_command_id = envelope.command_id;
|
||||||
snapshot.revision = snapshot.revision.saturating_add(1);
|
snapshot.revision = snapshot.revision.saturating_add(1);
|
||||||
true
|
accepted.push_back(AcceptedWorkerCommand {
|
||||||
|
envelope,
|
||||||
|
kind,
|
||||||
|
disposition: None,
|
||||||
|
});
|
||||||
|
WorkerCommandAdmission::Accepted
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn complete_command(
|
||||||
|
&self,
|
||||||
|
command_id: u64,
|
||||||
|
kind: WorkerCommandKind,
|
||||||
|
disposition: WorkerCommandDisposition,
|
||||||
|
) {
|
||||||
|
if !matches!(
|
||||||
|
disposition,
|
||||||
|
WorkerCommandDisposition::Accepted | WorkerCommandDisposition::InvalidState
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut accepted = self
|
||||||
|
.accepted_commands
|
||||||
|
.write()
|
||||||
|
.expect("worker command ledger lock poisoned; refusing command completion");
|
||||||
|
if let Some(command) = accepted
|
||||||
|
.iter_mut()
|
||||||
|
.find(|command| command.envelope.command_id == command_id && command.kind == kind)
|
||||||
|
{
|
||||||
|
command.disposition.get_or_insert(disposition);
|
||||||
|
}
|
||||||
|
while accepted
|
||||||
|
.iter()
|
||||||
|
.filter(|command| command.disposition.is_some())
|
||||||
|
.count()
|
||||||
|
> COMPLETED_COMMAND_RETENTION
|
||||||
|
{
|
||||||
|
let Some(index) = accepted
|
||||||
|
.iter()
|
||||||
|
.position(|command| command.disposition.is_some())
|
||||||
|
else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
accepted.remove(index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn command_result(
|
||||||
|
&self,
|
||||||
|
command_id: u64,
|
||||||
|
) -> Option<Option<WorkerCommandDisposition>> {
|
||||||
|
self.accepted_commands
|
||||||
|
.read()
|
||||||
|
.expect("worker command ledger lock poisoned")
|
||||||
|
.iter()
|
||||||
|
.find(|command| command.envelope.command_id == command_id)
|
||||||
|
.map(|command| command.disposition)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn snapshot(&self) -> WorkerStateSnapshot {
|
pub fn snapshot(&self) -> WorkerStateSnapshot {
|
||||||
@@ -190,9 +294,17 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn accepted_command_id_advances_the_snapshot_revision_atomically() {
|
fn accepted_command_identity_advances_revision_and_detects_reuse_conflicts() {
|
||||||
let state = test_state();
|
let state = test_state();
|
||||||
assert!(state.accept_command_id(9));
|
let envelope = WorkerCommandEnvelope {
|
||||||
|
command_id: 9,
|
||||||
|
expected_execution_generation: 7,
|
||||||
|
expected_worker_state_revision: 0,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
state.admit_command(envelope, WorkerCommandKind::Pause, true),
|
||||||
|
WorkerCommandAdmission::Accepted
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
state.snapshot(),
|
state.snapshot(),
|
||||||
WorkerStateSnapshot {
|
WorkerStateSnapshot {
|
||||||
@@ -202,7 +314,24 @@ mod tests {
|
|||||||
state: WorkerState::Idle,
|
state: WorkerState::Idle,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
assert!(!state.accept_command_id(9));
|
assert_eq!(state.command_result(9), Some(None));
|
||||||
|
state.complete_command(
|
||||||
|
9,
|
||||||
|
WorkerCommandKind::Pause,
|
||||||
|
WorkerCommandDisposition::Accepted,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.command_result(9),
|
||||||
|
Some(Some(WorkerCommandDisposition::Accepted))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.admit_command(envelope, WorkerCommandKind::Pause, true),
|
||||||
|
WorkerCommandAdmission::Retry
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
state.admit_command(envelope, WorkerCommandKind::Cancel, true),
|
||||||
|
WorkerCommandAdmission::Conflict
|
||||||
|
);
|
||||||
assert_eq!(state.snapshot().revision, 1);
|
assert_eq!(state.snapshot().revision, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1568,7 +1568,12 @@ pub struct WorkerSummary {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub tags: Vec<String>,
|
pub tags: Vec<String>,
|
||||||
pub workspace: WorkerWorkspaceSummary,
|
pub workspace: WorkerWorkspaceSummary,
|
||||||
|
/// Runtime catalog lifecycle compatibility state. Live foreground state, when
|
||||||
|
/// available, is carried separately in `worker_state`.
|
||||||
pub state: String,
|
pub state: String,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
#[cfg_attr(feature = "typescript", ts(optional))]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
pub last_seen_at: Option<String>,
|
pub last_seen_at: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub pinned: bool,
|
pub pinned: bool,
|
||||||
|
|||||||
@@ -243,7 +243,10 @@ pub struct WorkerSummary {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub tags: Vec<String>,
|
pub tags: Vec<String>,
|
||||||
pub workspace: WorkerWorkspaceSummary,
|
pub workspace: WorkerWorkspaceSummary,
|
||||||
|
/// Runtime catalog lifecycle compatibility state.
|
||||||
pub state: String,
|
pub state: String,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub worker_state: Option<protocol::WorkerStateSnapshot>,
|
||||||
pub last_seen_at: Option<String>,
|
pub last_seen_at: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub pinned: bool,
|
pub pinned: bool,
|
||||||
@@ -335,6 +338,7 @@ pub(crate) fn workspace_worker_summary(
|
|||||||
workspace_id: summary.workspace.workspace_id,
|
workspace_id: summary.workspace.workspace_id,
|
||||||
},
|
},
|
||||||
state: summary.state,
|
state: summary.state,
|
||||||
|
worker_state: summary.worker_state,
|
||||||
last_seen_at: summary.last_seen_at,
|
last_seen_at: summary.last_seen_at,
|
||||||
pinned: summary.pinned,
|
pinned: summary.pinned,
|
||||||
retention_state: summary.retention_state,
|
retention_state: summary.retention_state,
|
||||||
@@ -1998,6 +2002,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
workspace_id: summary.workspace_id.clone(),
|
workspace_id: summary.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(summary.status).to_string(),
|
state: embedded_worker_status_label(summary.status).to_string(),
|
||||||
|
worker_state: summary.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -2037,6 +2042,7 @@ impl EmbeddedWorkerRuntime {
|
|||||||
workspace_id: detail.workspace_id.clone(),
|
workspace_id: detail.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(detail.status).to_string(),
|
state: embedded_worker_status_label(detail.status).to_string(),
|
||||||
|
worker_state: detail.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -3340,6 +3346,7 @@ impl RemoteWorkerRuntime {
|
|||||||
workspace_id: summary.workspace_id.clone(),
|
workspace_id: summary.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(summary.status).to_string(),
|
state: embedded_worker_status_label(summary.status).to_string(),
|
||||||
|
worker_state: summary.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -3383,6 +3390,7 @@ impl RemoteWorkerRuntime {
|
|||||||
workspace_id: detail.workspace_id.clone(),
|
workspace_id: detail.workspace_id.clone(),
|
||||||
},
|
},
|
||||||
state: embedded_worker_status_label(detail.status).to_string(),
|
state: embedded_worker_status_label(detail.status).to_string(),
|
||||||
|
worker_state: detail.worker_state.clone(),
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -4730,6 +4738,7 @@ pub fn placeholder_worker(host_id: impl Into<String>) -> WorkerSummary {
|
|||||||
workspace_id: None,
|
workspace_id: None,
|
||||||
},
|
},
|
||||||
state: "unsupported".to_string(),
|
state: "unsupported".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
@@ -5251,6 +5260,7 @@ mod tests {
|
|||||||
workspace_id: None,
|
workspace_id: None,
|
||||||
},
|
},
|
||||||
state: "available".to_string(),
|
state: "available".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "transient".to_string(),
|
retention_state: "transient".to_string(),
|
||||||
|
|||||||
@@ -202,7 +202,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
|||||||
BrokerSubscriptionEvent::Event {
|
BrokerSubscriptionEvent::Event {
|
||||||
payload: SubscriptionEventPayload::WorkerUpserted { ref worker },
|
payload: SubscriptionEventPayload::WorkerUpserted { ref worker },
|
||||||
..
|
..
|
||||||
} if worker.state == SubscriptionWorkerState::Running
|
} if worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running
|
||||||
|
)),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap();
|
||||||
@@ -212,7 +221,18 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() {
|
|||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
snapshot,
|
snapshot,
|
||||||
SubscriptionSnapshot::Workers { workers }
|
SubscriptionSnapshot::Workers { workers }
|
||||||
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
if workers.iter().any(|worker| {
|
||||||
|
worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(
|
||||||
|
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
|
||||||
|
),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
|
})
|
||||||
));
|
));
|
||||||
drop(late);
|
drop(late);
|
||||||
|
|
||||||
@@ -337,8 +357,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(matches!(next_event(&mut subscription).await,
|
assert!(matches!(next_event(&mut subscription).await,
|
||||||
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
|
BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. }
|
||||||
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running));
|
if worker.runtime_id.as_deref() == Some("embedded-worker-runtime")
|
||||||
|
&& worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||||
|
protocol::WorkerRunState::Running
|
||||||
|
)),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)));
|
||||||
let mut late = broker
|
let mut late = broker
|
||||||
.subscribe(
|
.subscribe(
|
||||||
"embedded-worker-runtime",
|
"embedded-worker-runtime",
|
||||||
@@ -351,7 +381,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() {
|
|||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
snapshot,
|
snapshot,
|
||||||
SubscriptionSnapshot::Workers { workers }
|
SubscriptionSnapshot::Workers { workers }
|
||||||
if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running)
|
if workers.iter().any(|worker| {
|
||||||
|
worker.state == SubscriptionWorkerState::Idle
|
||||||
|
&& matches!(
|
||||||
|
worker.worker_state,
|
||||||
|
Some(protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Busy(
|
||||||
|
protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running)
|
||||||
|
),
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)
|
||||||
|
})
|
||||||
));
|
));
|
||||||
|
|
||||||
runtime
|
runtime
|
||||||
|
|||||||
@@ -15252,6 +15252,7 @@ fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> WorkerSummary
|
|||||||
singleton_key: None,
|
singleton_key: None,
|
||||||
tags: Vec::new(),
|
tags: Vec::new(),
|
||||||
state: "missing".to_string(),
|
state: "missing".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: Some(record.updated_at.clone()),
|
last_seen_at: Some(record.updated_at.clone()),
|
||||||
pinned: record.retention_state == "pinned",
|
pinned: record.retention_state == "pinned",
|
||||||
retention_state: record.retention_state.clone(),
|
retention_state: record.retention_state.clone(),
|
||||||
@@ -24990,6 +24991,7 @@ mod tests {
|
|||||||
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
||||||
},
|
},
|
||||||
state: "idle".to_string(),
|
state: "idle".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "normal".to_string(),
|
retention_state: "normal".to_string(),
|
||||||
@@ -25086,6 +25088,7 @@ mod tests {
|
|||||||
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
workspace_id: Some(TEST_WORKSPACE_ID.to_string()),
|
||||||
},
|
},
|
||||||
state: "idle".to_string(),
|
state: "idle".to_string(),
|
||||||
|
worker_state: None,
|
||||||
last_seen_at: None,
|
last_seen_at: None,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: "normal".to_string(),
|
retention_state: "normal".to_string(),
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ command_id: number, expected_execution_generation: number, expected_worker_state
|
|||||||
|
|
||||||
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
|
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
|
||||||
|
|
||||||
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "invalid_state";
|
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
|
||||||
|
|
||||||
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
|
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
|
||||||
/**
|
/**
|
||||||
@@ -233,7 +233,16 @@ resource_key?: string | null,
|
|||||||
/**
|
/**
|
||||||
* Producer-owned monotonic revision for this Worker subject.
|
* Producer-owned monotonic revision for this Worker subject.
|
||||||
*/
|
*/
|
||||||
subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
subject_revision: number,
|
||||||
|
/**
|
||||||
|
* Latest revisioned foreground state observed from the Worker. This remains
|
||||||
|
* absent until an authoritative Worker snapshot/event has been applied.
|
||||||
|
*/
|
||||||
|
worker_state?: WorkerStateSnapshot | null,
|
||||||
|
/**
|
||||||
|
* Runtime catalog lifecycle compatibility projection; not foreground-state authority.
|
||||||
|
*/
|
||||||
|
state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
||||||
/**
|
/**
|
||||||
* Workspace-facing Repository key. Runtime producers leave this unset and
|
* Workspace-facing Repository key. Runtime producers leave this unset and
|
||||||
* Workspace Server projections replace `repository_id` with this field.
|
* Workspace Server projections replace `repository_id` with this field.
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import type {
|
|||||||
Event as PodProtocolEvent,
|
Event as PodProtocolEvent,
|
||||||
Method as PodProtocolMethod,
|
Method as PodProtocolMethod,
|
||||||
Segment as PodProtocolSegment,
|
Segment as PodProtocolSegment,
|
||||||
|
WorkerStateSnapshot,
|
||||||
} from "$lib/generated/protocol";
|
} from "$lib/generated/protocol";
|
||||||
import type {
|
import type {
|
||||||
GitCommitSummary as SharedGitCommitSummary,
|
GitCommitSummary as SharedGitCommitSummary,
|
||||||
@@ -99,6 +100,7 @@ export type Worker = {
|
|||||||
tags: string[];
|
tags: string[];
|
||||||
workspace: { visibility: string; identity: string };
|
workspace: { visibility: string; identity: string };
|
||||||
state: string;
|
state: string;
|
||||||
|
worker_state?: WorkerStateSnapshot | null;
|
||||||
pinned?: boolean;
|
pinned?: boolean;
|
||||||
retention_state?: string;
|
retention_state?: string;
|
||||||
last_seen_at?: string | null;
|
last_seen_at?: string | null;
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import type { WorkerStateSnapshot } from "$lib/generated/protocol";
|
||||||
|
|
||||||
|
export function liveWorkerState(worker: {
|
||||||
|
state: string;
|
||||||
|
worker_state?: WorkerStateSnapshot | null;
|
||||||
|
}): string {
|
||||||
|
const state = worker.worker_state?.state;
|
||||||
|
if (!state) return worker.state === "stopped" ? "stopped" : "unknown";
|
||||||
|
if (state.kind === "idle") return "idle";
|
||||||
|
if (state.state.kind === "maintenance") return "running";
|
||||||
|
return state.state.state === "paused" ? "paused" : "running";
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void {
|
|||||||
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
import { liveWorkerState } from './worker-state';
|
||||||
import {
|
import {
|
||||||
applyWorkspaceWorkersFrame,
|
applyWorkspaceWorkersFrame,
|
||||||
createWorkspaceWorkersProjection,
|
createWorkspaceWorkersProjection,
|
||||||
@@ -33,6 +34,22 @@ function worker(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
|
||||||
|
const active = worker('runtime-a', 'worker-1', 1);
|
||||||
|
active.worker_state = {
|
||||||
|
execution_generation: 4,
|
||||||
|
revision: 2,
|
||||||
|
last_command_id: 1,
|
||||||
|
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
|
||||||
|
};
|
||||||
|
assertEquals(liveWorkerState(active), 'paused');
|
||||||
|
|
||||||
|
const unavailable = worker('runtime-a', 'worker-2', 1);
|
||||||
|
assertEquals(liveWorkerState(unavailable), 'unknown');
|
||||||
|
unavailable.state = 'stopped';
|
||||||
|
assertEquals(liveWorkerState(unavailable), 'stopped');
|
||||||
|
});
|
||||||
|
|
||||||
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
|
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
|
||||||
const projection = createWorkspaceWorkersProjection();
|
const projection = createWorkspaceWorkersProjection();
|
||||||
const frame: SubscriptionFrame = {
|
const frame: SubscriptionFrame = {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
applyWorkspaceWorkersFrame,
|
applyWorkspaceWorkersFrame,
|
||||||
createWorkspaceWorkersProjection,
|
createWorkspaceWorkersProjection,
|
||||||
} from './worker-subscription-model';
|
} from './worker-subscription-model';
|
||||||
|
import { liveWorkerState } from './worker-state';
|
||||||
import { compareWorkersForSidebar } from './workers';
|
import { compareWorkersForSidebar } from './workers';
|
||||||
import type { Worker } from './types';
|
import type { Worker } from './types';
|
||||||
|
|
||||||
@@ -86,7 +87,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker {
|
|||||||
profile: worker.profile ?? null,
|
profile: worker.profile ?? null,
|
||||||
tags: [],
|
tags: [],
|
||||||
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
|
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
|
||||||
state: worker.state,
|
state: liveWorkerState(worker),
|
||||||
|
worker_state: worker.worker_state,
|
||||||
pinned: false,
|
pinned: false,
|
||||||
retention_state: 'transient',
|
retention_state: 'transient',
|
||||||
implementation: {
|
implementation: {
|
||||||
|
|||||||
+16
-44
@@ -52,11 +52,7 @@
|
|||||||
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
|
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
|
||||||
import { workspaceApiPath } from "$lib/workspace/api/http";
|
import { workspaceApiPath } from "$lib/workspace/api/http";
|
||||||
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
|
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
|
||||||
import type {
|
import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types";
|
||||||
Diagnostic,
|
|
||||||
Worker,
|
|
||||||
PodProtocolEvent,
|
|
||||||
} from "$lib/workspace/sidebar/types";
|
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
data: {
|
data: {
|
||||||
@@ -207,7 +203,6 @@
|
|||||||
);
|
);
|
||||||
let pendingObservationEvents: ConsoleEventInput[] = [];
|
let pendingObservationEvents: ConsoleEventInput[] = [];
|
||||||
let protocolEventSequence = 0;
|
let protocolEventSequence = 0;
|
||||||
let pendingObservedStates: Array<string | null> = [];
|
|
||||||
let pendingStreamDiagnostics: Diagnostic[] = [];
|
let pendingStreamDiagnostics: Diagnostic[] = [];
|
||||||
let observationFlushHandle: number | null = null;
|
let observationFlushHandle: number | null = null;
|
||||||
let nextReloadToken = 0;
|
let nextReloadToken = 0;
|
||||||
@@ -249,7 +244,9 @@
|
|||||||
const diagnostics = $derived(
|
const diagnostics = $derived(
|
||||||
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
|
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
|
||||||
);
|
);
|
||||||
const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading");
|
const workerState = $derived(
|
||||||
|
liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"),
|
||||||
|
);
|
||||||
const workerRunning = $derived(workerState === "running");
|
const workerRunning = $derived(workerState === "running");
|
||||||
const workerPaused = $derived(workerState === "paused");
|
const workerPaused = $derived(workerState === "paused");
|
||||||
const composerEditable = $derived(protocolState === "open" && !sending);
|
const composerEditable = $derived(protocolState === "open" && !sending);
|
||||||
@@ -343,7 +340,6 @@
|
|||||||
observationFlushHandle = null;
|
observationFlushHandle = null;
|
||||||
}
|
}
|
||||||
pendingObservationEvents = [];
|
pendingObservationEvents = [];
|
||||||
pendingObservedStates = [];
|
|
||||||
pendingStreamDiagnostics = [];
|
pendingStreamDiagnostics = [];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -359,18 +355,15 @@
|
|||||||
function flushObservationBatch() {
|
function flushObservationBatch() {
|
||||||
observationFlushHandle = null;
|
observationFlushHandle = null;
|
||||||
const eventBatch = pendingObservationEvents;
|
const eventBatch = pendingObservationEvents;
|
||||||
const stateBatch = pendingObservedStates;
|
|
||||||
const diagnosticBatch = pendingStreamDiagnostics;
|
const diagnosticBatch = pendingStreamDiagnostics;
|
||||||
pendingObservationEvents = [];
|
pendingObservationEvents = [];
|
||||||
pendingObservedStates = [];
|
|
||||||
pendingStreamDiagnostics = [];
|
pendingStreamDiagnostics = [];
|
||||||
|
|
||||||
if (eventBatch.length > 0) {
|
if (eventBatch.length > 0) {
|
||||||
const latestState = stateBatch.findLast((state) => state !== null);
|
|
||||||
if (latestState) {
|
|
||||||
liveWorkerState = latestState;
|
|
||||||
}
|
|
||||||
consoleProjection = consoleProjector.append(eventBatch);
|
consoleProjection = consoleProjector.append(eventBatch);
|
||||||
|
liveWorkerState = consoleProjection.status === "shutdown"
|
||||||
|
? "shutdown"
|
||||||
|
: workerStateFromSnapshot(consoleProjection.workerState);
|
||||||
advanceEventObservedAtVersion();
|
advanceEventObservedAtVersion();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -407,7 +400,6 @@
|
|||||||
event: payload,
|
event: payload,
|
||||||
observedAtMs,
|
observedAtMs,
|
||||||
});
|
});
|
||||||
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
|
|
||||||
scheduleObservationFlush();
|
scheduleObservationFlush();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -925,36 +917,16 @@
|
|||||||
handleComposerSubmit();
|
handleComposerSubmit();
|
||||||
}
|
}
|
||||||
|
|
||||||
function workerStateFromProtocolEvent(
|
function workerStateFromSnapshot(
|
||||||
event: PodProtocolEvent,
|
snapshot: ConsoleProjection["workerState"],
|
||||||
): string | null {
|
): string | null {
|
||||||
switch (event.event) {
|
if (!snapshot) return null;
|
||||||
case "snapshot":
|
return snapshot.state.kind === "idle"
|
||||||
return event.data.state.state.kind === "idle"
|
? "idle"
|
||||||
? "idle"
|
: snapshot.state.state.kind === "run" &&
|
||||||
: event.data.state.state.state.kind === "run" &&
|
snapshot.state.state.state === "paused"
|
||||||
event.data.state.state.state.state === "paused"
|
? "paused"
|
||||||
? "paused"
|
: "running";
|
||||||
: "running";
|
|
||||||
case "worker_state":
|
|
||||||
return event.data.snapshot.state.kind === "idle"
|
|
||||||
? "idle"
|
|
||||||
: event.data.snapshot.state.state.kind === "run" &&
|
|
||||||
event.data.snapshot.state.state.state === "paused"
|
|
||||||
? "paused"
|
|
||||||
: "running";
|
|
||||||
case "command_acknowledged":
|
|
||||||
return event.data.acknowledgement.state.state.kind === "idle"
|
|
||||||
? "idle"
|
|
||||||
: event.data.acknowledgement.state.state.state.kind === "run" &&
|
|
||||||
event.data.acknowledgement.state.state.state.state === "paused"
|
|
||||||
? "paused"
|
|
||||||
: "running";
|
|
||||||
case "shutdown":
|
|
||||||
return "shutdown";
|
|
||||||
default:
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function connectProtocolTransport(
|
function connectProtocolTransport(
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
import { workerHref } from '$lib/workspace/resource-links';
|
import { workerHref } from '$lib/workspace/resource-links';
|
||||||
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
|
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
|
||||||
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
|
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
|
||||||
|
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||||
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
|
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
|
||||||
import type { PageProps } from './$types';
|
import type { PageProps } from './$types';
|
||||||
|
|
||||||
@@ -136,7 +137,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
function workerStatus(worker: Worker): string {
|
function workerStatus(worker: Worker): string {
|
||||||
return worker.state;
|
return liveWorkerState(worker);
|
||||||
}
|
}
|
||||||
|
|
||||||
function workerProfile(worker: Worker): string {
|
function workerProfile(worker: Worker): string {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { workspaceRoute } from '$lib/workspace/api/http';
|
import { workspaceRoute } from '$lib/workspace/api/http';
|
||||||
|
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||||
import type { PageData } from './$types';
|
import type { PageData } from './$types';
|
||||||
let { data }: { data: PageData } = $props();
|
let { data }: { data: PageData } = $props();
|
||||||
</script>
|
</script>
|
||||||
@@ -24,7 +25,7 @@
|
|||||||
>Open console</a>
|
>Open console</a>
|
||||||
</header>
|
</header>
|
||||||
<dl class="resource-meta">
|
<dl class="resource-meta">
|
||||||
<dt>Status</dt><dd>{data.worker.state}</dd>
|
<dt>Status</dt><dd>{liveWorkerState(data.worker)}</dd>
|
||||||
<dt>Profile</dt><dd>{data.worker.profile}</dd>
|
<dt>Profile</dt><dd>{data.worker.profile}</dd>
|
||||||
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
|
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
|
||||||
</dl>
|
</dl>
|
||||||
|
|||||||
Reference in New Issue
Block a user