Implement transactional Worker restore outcomes

This commit is contained in:
2026-09-18 00:08:00 +09:00
parent 5ae1bab4ee
commit 1d7cd9339c
11 changed files with 739 additions and 187 deletions
+1
View File
@@ -17,6 +17,7 @@ pub use workspace_api::{
WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate, WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate,
WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption, WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption,
WorkerOperationState as BackendWorkerOperationState, WorkerOperationState as BackendWorkerOperationState,
WorkerRestoreState as BackendWorkerRestoreState,
WorkerRestoreResponse as BackendWorkerRestoreResponse, WorkerRestoreResponse as BackendWorkerRestoreResponse,
WorkerRestoreResult as BackendWorkerRestoreResult, WorkerSummary as BackendWorkerSummary, WorkerRestoreResult as BackendWorkerRestoreResult, WorkerSummary as BackendWorkerSummary,
WorkerWorkspaceSummary as BackendWorkerWorkspaceSummary, WorkerWorkspaceSummary as BackendWorkerWorkspaceSummary,
+1 -1
View File
@@ -27,7 +27,7 @@ pub use backend_runtime::{
BackendWorkerCapabilitySummary, BackendWorkerImplementationSummary, BackendWorkerLaunchOptions, BackendWorkerCapabilitySummary, BackendWorkerImplementationSummary, BackendWorkerLaunchOptions,
BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption, BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption,
BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse, BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse,
BackendWorkerRestoreResult, BackendWorkerSummary, BackendWorkerWorkspaceSummary, BackendWorkerRestoreResult, BackendWorkerRestoreState, BackendWorkerSummary, BackendWorkerWorkspaceSummary,
BackendWorkingDirectorySummary, connect_backend_runtime, create_backend_worker, BackendWorkingDirectorySummary, connect_backend_runtime, create_backend_worker,
get_backend_worker_launch_options, list_backend_stopped_workers, list_backend_workers, get_backend_worker_launch_options, list_backend_stopped_workers, list_backend_workers,
restore_backend_worker, restore_backend_worker,
+10 -9
View File
@@ -3,7 +3,7 @@ use std::io;
use std::time::Duration; use std::time::Duration;
use client::{ use client::{
BackendRuntimeListTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse, BackendRuntimeListTarget, BackendWorkerRestoreResponse, BackendWorkerRestoreState,
BackendWorkerSummary, list_backend_stopped_workers, list_backend_workers, BackendWorkerSummary, list_backend_stopped_workers, list_backend_workers,
restore_backend_worker, restore_backend_worker,
}; };
@@ -110,7 +110,7 @@ pub(crate) async fn run(
} }
fn restored_worker(response: BackendWorkerRestoreResponse) -> Result<BackendWorkerSummary, String> { fn restored_worker(response: BackendWorkerRestoreResponse) -> Result<BackendWorkerSummary, String> {
if response.result.state != BackendWorkerOperationState::Accepted { if response.result.state != BackendWorkerRestoreState::Accepted {
let diagnostics = response let diagnostics = response
.result .result
.diagnostics .diagnostics
@@ -119,9 +119,10 @@ fn restored_worker(response: BackendWorkerRestoreResponse) -> Result<BackendWork
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join("; "); .join("; ");
let state = match response.result.state { let state = match response.result.state {
BackendWorkerOperationState::Accepted => unreachable!(), BackendWorkerRestoreState::Accepted => unreachable!(),
BackendWorkerOperationState::Rejected => "rejected", BackendWorkerRestoreState::Rejected => "rejected",
BackendWorkerOperationState::Unsupported => "unsupported", BackendWorkerRestoreState::RolledBack => "rolled back",
BackendWorkerRestoreState::ReconciliationRequired => "reconciliation required",
}; };
return Err(if diagnostics.is_empty() { return Err(if diagnostics.is_empty() {
format!("restore was {state} without a diagnostic") format!("restore was {state} without a diagnostic")
@@ -494,7 +495,7 @@ mod tests {
} }
fn restore_response( fn restore_response(
state: BackendWorkerOperationState, state: BackendWorkerRestoreState,
worker: Option<BackendWorkerSummary>, worker: Option<BackendWorkerSummary>,
diagnostics: Vec<BackendDiagnostic>, diagnostics: Vec<BackendDiagnostic>,
) -> BackendWorkerRestoreResponse { ) -> BackendWorkerRestoreResponse {
@@ -513,7 +514,7 @@ mod tests {
#[test] #[test]
fn rejected_restore_surfaces_diagnostic_instead_of_attaching_selected_worker() { fn rejected_restore_surfaces_diagnostic_instead_of_attaching_selected_worker() {
let error = restored_worker(restore_response( let error = restored_worker(restore_response(
BackendWorkerOperationState::Rejected, BackendWorkerRestoreState::Rejected,
None, None,
vec![BackendDiagnostic { vec![BackendDiagnostic {
code: "working_directory_not_found".to_string(), code: "working_directory_not_found".to_string(),
@@ -532,7 +533,7 @@ mod tests {
#[test] #[test]
fn accepted_restore_requires_returned_worker_snapshot() { fn accepted_restore_requires_returned_worker_snapshot() {
let error = restored_worker(restore_response( let error = restored_worker(restore_response(
BackendWorkerOperationState::Accepted, BackendWorkerRestoreState::Accepted,
None, None,
Vec::new(), Vec::new(),
)) ))
@@ -545,7 +546,7 @@ mod tests {
fn accepted_restore_returns_authoritative_worker_snapshot() { fn accepted_restore_returns_authoritative_worker_snapshot() {
let worker = worker("runtime-a", "worker-a", Some("builtin:companion")); let worker = worker("runtime-a", "worker-a", Some("builtin:companion"));
let restored = restored_worker(restore_response( let restored = restored_worker(restore_response(
BackendWorkerOperationState::Accepted, BackendWorkerRestoreState::Accepted,
Some(worker.clone()), Some(worker.clone()),
Vec::new(), Vec::new(),
)) ))
+30
View File
@@ -286,7 +286,21 @@ pub enum WorkerExecutionSpawnResult {
worker_state: protocol::WorkerStateSnapshot, worker_state: protocol::WorkerStateSnapshot,
working_directory: Option<WorkingDirectoryStatus>, working_directory: Option<WorkingDirectoryStatus>,
}, },
/// A read-only preflight rejected the operation before live work started.
Rejected(WorkerExecutionResult), Rejected(WorkerExecutionResult),
/// Live work started, failed, and all operation-owned resources were joined/released.
RolledBack(WorkerExecutionResult),
/// The operation crossed the live side-effect boundary and could not prove
/// either commit or complete cleanup. Optional handle/state fields preserve
/// concrete execution evidence for retry reconciliation.
ReconciliationRequired {
result: WorkerExecutionResult,
handle: Option<WorkerExecutionHandle>,
worker_state: Option<protocol::WorkerStateSnapshot>,
working_directory: Option<WorkingDirectoryStatus>,
},
/// Legacy spawn failure. Restore implementations must use `RolledBack` or
/// `ReconciliationRequired` after crossing their live side-effect boundary.
Errored(WorkerExecutionResult), Errored(WorkerExecutionResult),
} }
@@ -316,6 +330,15 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult; fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult;
/// Read-only restore validation performed before any live controller,
/// bridge, task, or external registration is created.
fn preflight_restore(
&self,
_request: &WorkerExecutionRestoreRequest,
) -> Result<(), WorkerExecutionResult> {
Ok(())
}
fn restore_worker( fn restore_worker(
&self, &self,
_request: WorkerExecutionRestoreRequest, _request: WorkerExecutionRestoreRequest,
@@ -509,6 +532,13 @@ impl WorkerExecutionBackendRef {
self.backend.spawn_worker(request) self.backend.spawn_worker(request)
} }
pub(crate) fn preflight_restore(
&self,
request: &WorkerExecutionRestoreRequest,
) -> Result<(), WorkerExecutionResult> {
self.backend.preflight_restore(request)
}
pub(crate) fn restore_worker( pub(crate) fn restore_worker(
&self, &self,
request: WorkerExecutionRestoreRequest, request: WorkerExecutionRestoreRequest,
+30 -7
View File
@@ -60,6 +60,7 @@ use protocol::subscription::{
SubscriptionTerminationCode, SubscriptionTerminationCode,
}; };
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use workspace_api::WorkerRestoreState;
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt; use std::fmt;
use std::net::SocketAddr; use std::net::SocketAddr;
@@ -420,6 +421,20 @@ pub struct RuntimeHttpWorkerResponse {
pub worker: WorkerDetail, pub worker: WorkerDetail,
} }
/// Explicit restore outcome. This envelope is returned for every reachable
/// Runtime restore attempt, including preflight rejection and uncertain commit.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RuntimeHttpWorkerRestoreResponse {
pub state: WorkerRestoreState,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub worker: Option<WorkerDetail>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reason_code: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
}
/// Replace the Workspace API binding for an existing Worker. /// Replace the Workspace API binding for an existing Worker.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeHttpWorkerWorkspaceApiRequest { pub struct RuntimeHttpWorkerWorkspaceApiRequest {
@@ -1199,14 +1214,21 @@ async fn restore_worker(
State(state): State<RuntimeHttpState>, State(state): State<RuntimeHttpState>,
auth: Option<Extension<RuntimeAuthContext>>, auth: Option<Extension<RuntimeAuthContext>>,
Path(worker_id): Path<String>, Path(worker_id): Path<String>,
) -> RestResult<RuntimeHttpWorkerResponse> { ) -> RestResult<RuntimeHttpWorkerRestoreResponse> {
let worker_ref = worker_ref_for(&state.runtime, worker_id)?; let worker_ref = worker_ref_for(&state.runtime, worker_id)?;
let worker = match auth_workspace_scope(&state, auth.as_ref())? { let result = match auth_workspace_scope(&state, auth.as_ref())? {
Some(scope) => state.runtime.restore_worker_scoped(&scope, &worker_ref), Some(scope) => state
None => state.runtime.restore_worker(&worker_ref), .runtime
.restore_worker_operation_scoped(&scope, &worker_ref),
None => state.runtime.restore_worker_operation(&worker_ref),
} }
.map_err(RuntimeHttpRestError::runtime)?; .map_err(RuntimeHttpRestError::runtime)?;
Ok(Json(RuntimeHttpWorkerResponse { worker })) Ok(Json(RuntimeHttpWorkerRestoreResponse {
state: result.state,
worker: result.worker,
reason_code: result.reason_code,
message: result.message,
}))
} }
#[cfg(feature = "ws-server")] #[cfg(feature = "ws-server")]
@@ -3322,8 +3344,9 @@ mod tests {
) )
.await; .await;
assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.status(), StatusCode::OK);
let restored: RuntimeHttpWorkerResponse = read_json(response).await; let restored: RuntimeHttpWorkerRestoreResponse = read_json(response).await;
assert_eq!(restored.worker.status, WorkerStatus::Idle); assert_eq!(restored.state, WorkerRestoreState::Accepted);
assert_eq!(restored.worker.unwrap().status, WorkerStatus::Idle);
let response = authed_empty_request( let response = authed_empty_request(
app.clone(), app.clone(),
+408 -114
View File
@@ -54,6 +54,7 @@ use std::sync::{Arc, Mutex, MutexGuard, Weak};
use tokio::sync::broadcast; use tokio::sync::broadcast;
use tokio::sync::mpsc; use tokio::sync::mpsc;
use uuid::Uuid; use uuid::Uuid;
use workspace_api::WorkerRestoreState;
/// Workspace-scoped Runtime authorization context supplied by a trusted backend. /// Workspace-scoped Runtime authorization context supplied by a trusted backend.
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
@@ -83,6 +84,47 @@ enum WorkerRestoreMode {
Automatic, Automatic,
} }
/// Runtime-internal restore result consumed by both embedded and HTTP
/// providers. Diagnostics are stable and path-free; detailed backend errors
/// remain in Runtime logs rather than crossing the API boundary.
#[derive(Clone, Debug)]
pub struct RuntimeWorkerRestoreResult {
pub state: WorkerRestoreState,
pub worker: Option<WorkerDetail>,
pub reason_code: Option<String>,
pub message: Option<String>,
}
impl RuntimeWorkerRestoreResult {
fn accepted(worker: WorkerDetail) -> Self {
Self {
state: WorkerRestoreState::Accepted,
worker: Some(worker),
reason_code: None,
message: None,
}
}
fn failed(state: WorkerRestoreState, reason_code: &'static str, message: &'static str) -> Self {
Self {
state,
worker: None,
reason_code: Some(reason_code.to_string()),
message: Some(message.to_string()),
}
}
fn into_legacy_result(self) -> Result<WorkerDetail, RuntimeError> {
match (self.state, self.worker) {
(WorkerRestoreState::Accepted, Some(worker)) => Ok(worker),
_ => Err(RuntimeError::InvalidRequest(
self.message
.unwrap_or_else(|| "Worker restore did not commit".to_string()),
)),
}
}
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct RuntimeSubscriptionUpdate { pub struct RuntimeSubscriptionUpdate {
pub subject_revision: u64, pub subject_revision: u64,
@@ -1047,6 +1089,7 @@ impl Runtime {
working_directory, working_directory,
} => (handle, worker_state, working_directory), } => (handle, worker_state, working_directory),
WorkerExecutionSpawnResult::Rejected(result) WorkerExecutionSpawnResult::Rejected(result)
| WorkerExecutionSpawnResult::RolledBack(result)
| WorkerExecutionSpawnResult::Errored(result) => { | WorkerExecutionSpawnResult::Errored(result) => {
self.rollback_failed_create(&worker_ref)?; self.rollback_failed_create(&worker_ref)?;
return Err(RuntimeError::WorkerExecutionRejected { return Err(RuntimeError::WorkerExecutionRejected {
@@ -1057,6 +1100,16 @@ impl Runtime {
result, result,
}); });
} }
WorkerExecutionSpawnResult::ReconciliationRequired { result, .. } => {
self.rollback_failed_create(&worker_ref)?;
return Err(RuntimeError::WorkerExecutionRejected {
worker_id: worker_ref.worker_id.clone(),
operation: result.operation,
outcome: result.outcome,
message: result.message_or_default(),
result,
});
}
}; };
if let Some(mut initial_input) = { if let Some(mut initial_input) = {
@@ -1377,7 +1430,16 @@ impl Runtime {
scope: &RuntimeWorkspaceScope, scope: &RuntimeWorkspaceScope,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
) -> Result<WorkerDetail, RuntimeError> { ) -> Result<WorkerDetail, RuntimeError> {
self.restore_worker_with_scope(worker_ref, Some(scope)) self.restore_worker_operation_scoped(scope, worker_ref)?
.into_legacy_result()
}
pub fn restore_worker_operation_scoped(
&self,
scope: &RuntimeWorkspaceScope,
worker_ref: &WorkerRef,
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
self.restore_worker_operation_with_scope(worker_ref, Some(scope))
} }
/// Attach a live execution to a persisted Worker definition. /// Attach a live execution to a persisted Worker definition.
@@ -1387,14 +1449,21 @@ impl Runtime {
/// converges on its already-installed execution rather than spawning another /// converges on its already-installed execution rather than spawning another
/// controller. /// controller.
pub fn restore_worker(&self, worker_ref: &WorkerRef) -> Result<WorkerDetail, RuntimeError> { pub fn restore_worker(&self, worker_ref: &WorkerRef) -> Result<WorkerDetail, RuntimeError> {
self.restore_worker_with_scope(worker_ref, None) self.restore_worker_operation(worker_ref)?.into_legacy_result()
} }
fn restore_worker_with_scope( pub fn restore_worker_operation(
&self,
worker_ref: &WorkerRef,
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
self.restore_worker_operation_with_scope(worker_ref, None)
}
fn restore_worker_operation_with_scope(
&self, &self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
scope: Option<&RuntimeWorkspaceScope>, scope: Option<&RuntimeWorkspaceScope>,
) -> Result<WorkerDetail, RuntimeError> { ) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?;
let _operation_guard = operation_lock let _operation_guard = operation_lock
.lock() .lock()
@@ -1409,58 +1478,61 @@ impl Runtime {
&self, &self,
worker_ref: &WorkerRef, worker_ref: &WorkerRef,
mode: WorkerRestoreMode, mode: WorkerRestoreMode,
) -> Result<WorkerDetail, RuntimeError> { ) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
let (backend, request) = { let (backend, request) = {
let mut state = self.lock()?; let state = self.lock()?;
state.ensure_running()?; state.ensure_running()?;
let (worker_request, previous_working_directory) = { let worker = state.worker(worker_ref)?;
let worker = state.worker(worker_ref)?; if worker.execution_bound && !worker.execution_metadata_available {
if worker.execution_handle.is_some() { return Ok(RuntimeWorkerRestoreResult::failed(
if worker.status.is_active() { WorkerRestoreState::ReconciliationRequired,
return Ok(worker.detail()); "worker_restore_reconciliation_pending",
} "Worker restore cleanup or reconciliation is still pending",
return Err(RuntimeError::WorkerExecutionUnavailable { ));
worker_id: worker_ref.worker_id,
message: "previous execution cleanup has not completed".to_string(),
});
}
match mode {
WorkerRestoreMode::Explicit
if worker.execution_metadata_available
&& worker.status != WorkerStatus::Stopped =>
{
return Err(RuntimeError::InvalidRequest(format!(
"worker {} is not stopped",
worker_ref.worker_id
)));
}
WorkerRestoreMode::Automatic
if !worker.status.is_active()
|| worker.restore_intent != WorkerRestoreIntent::Automatic =>
{
return Ok(worker.detail());
}
_ => {}
}
let request = worker.request.clone().ok_or_else(|| {
RuntimeError::WorkerExecutionUnavailable {
worker_id: worker.worker_id,
message: "persisted Worker restore request is unavailable".to_string(),
}
})?;
(request, worker.working_directory.clone())
};
let backend = state.execution_backend.clone().ok_or_else(|| {
RuntimeError::WorkerExecutionUnavailable {
worker_id: worker_ref.worker_id,
message: "runtime has no execution backend".to_string(),
}
})?;
{
let worker = state.worker_mut(worker_ref)?;
worker.execution_bound = true;
} }
state.persist_worker(&worker_ref.worker_id)?; if worker.execution_handle.is_some() {
if worker.status.is_active() {
return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail()));
}
return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_cleanup_pending",
"Worker restore cleanup or reconciliation is still pending",
));
}
match mode {
WorkerRestoreMode::Explicit
if worker.execution_metadata_available
&& worker.status != WorkerStatus::Stopped =>
{
return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::Rejected,
"worker_restore_not_stopped",
"Worker restore requires a stopped Worker",
));
}
WorkerRestoreMode::Automatic
if !worker.status.is_active()
|| worker.restore_intent != WorkerRestoreIntent::Automatic =>
{
return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail()));
}
_ => {}
}
let Some(worker_request) = worker.request.clone() else {
return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::Rejected,
"worker_restore_metadata_unavailable",
"Persisted Worker restore metadata is unavailable",
));
};
let Some(backend) = state.execution_backend.clone() else {
return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::Rejected,
"worker_restore_backend_unavailable",
"Runtime has no Worker execution backend",
));
};
let workspace_scope = worker_request.workspace_api.as_ref().and_then(|api| { let workspace_scope = worker_request.workspace_api.as_ref().and_then(|api| {
state state
.workspace_owners .workspace_owners
@@ -1472,13 +1544,21 @@ impl Runtime {
request: worker_request, request: worker_request,
workspace_scope, workspace_scope,
context: self.execution_context(worker_ref.clone()), context: self.execution_context(worker_ref.clone()),
previous_working_directory, previous_working_directory: worker.working_directory.clone(),
working_directory: None, working_directory: None,
config_bundle: None, config_bundle: None,
}; };
(backend, request) (backend, request)
}; };
if let Err(_result) = backend.preflight_restore(&request) {
return Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::Rejected,
"worker_restore_preflight_rejected",
"Worker restore preflight rejected the current request",
));
}
match backend.restore_worker(request) { match backend.restore_worker(request) {
WorkerExecutionSpawnResult::Connected { WorkerExecutionSpawnResult::Connected {
handle, handle,
@@ -1488,30 +1568,117 @@ impl Runtime {
let commit = self.commit_restored_worker_execution( let commit = self.commit_restored_worker_execution(
worker_ref, worker_ref,
handle.clone(), handle.clone(),
worker_state, worker_state.clone(),
WorkerStatus::Idle, WorkerStatus::Idle,
working_directory, working_directory.clone(),
); );
if let Err(error) = commit { match commit {
self.cleanup_failed_restore(&backend, worker_ref, &handle)?; Ok(worker) => Ok(RuntimeWorkerRestoreResult::accepted(worker)),
return Err(error); Err(error) => match self.cleanup_failed_restore(&backend, worker_ref, &handle) {
Ok(()) => {
tracing::warn!(
worker_id = %worker_ref.worker_id,
error = %error,
"Worker restore commit failed and was rolled back"
);
Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::RolledBack,
"worker_restore_commit_rolled_back",
"Worker restore commit failed and cleanup completed",
))
}
Err(cleanup_error) => {
tracing::error!(
worker_id = %worker_ref.worker_id,
error = %error,
cleanup_error = %cleanup_error,
"Worker restore requires reconciliation after commit and cleanup failures"
);
if let Err(retain_error) = self.retain_restore_execution_evidence(
worker_ref,
handle,
worker_state,
working_directory,
) {
tracing::error!(
worker_id = %worker_ref.worker_id,
error = %retain_error,
"failed to persist restore execution evidence; in-process reconciliation remains required"
);
}
Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_reconciliation_required",
"Worker restore result is uncertain; reread or retry the Worker restore",
))
}
},
} }
self.worker_detail(worker_ref)
} }
WorkerExecutionSpawnResult::Rejected(result) WorkerExecutionSpawnResult::Rejected(_result) => {
| WorkerExecutionSpawnResult::Errored(result) => { Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::Rejected,
"worker_restore_rejected",
"Worker restore was rejected before live work started",
))
}
WorkerExecutionSpawnResult::RolledBack(result) => {
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
self.lock()?
.record_restore_failure(worker_ref, result)?;
Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::RolledBack,
"worker_restore_rolled_back",
"Worker restore failed after live work started; cleanup completed",
))
}
WorkerExecutionSpawnResult::ReconciliationRequired {
result: _result,
handle,
worker_state,
working_directory,
} => {
if let (Some(handle), Some(worker_state)) = (handle, worker_state) {
if let Err(retain_error) = self.retain_restore_execution_evidence(
worker_ref,
handle,
worker_state,
working_directory,
) {
tracing::error!(
worker_id = %worker_ref.worker_id,
error = %retain_error,
"failed to persist restore execution evidence; in-process reconciliation remains required"
);
}
} else if let Err(retain_error) =
self.retain_restore_reconciliation_pending(worker_ref)
{ {
self.lock()? tracing::error!(
.record_restore_failure(worker_ref, result.clone())?; worker_id = %worker_ref.worker_id,
error = %retain_error,
"failed to persist pending restore reconciliation marker"
);
} }
Err(RuntimeError::WorkerExecutionRejected { Ok(RuntimeWorkerRestoreResult::failed(
worker_id: worker_ref.worker_id, WorkerRestoreState::ReconciliationRequired,
operation: result.operation, "worker_restore_reconciliation_required",
outcome: result.outcome, "Worker restore result is uncertain; reread or retry the Worker restore",
message: result.message_or_default(), ))
result, }
}) WorkerExecutionSpawnResult::Errored(_result) => {
if let Err(retain_error) = self.retain_restore_reconciliation_pending(worker_ref) {
tracing::error!(
worker_id = %worker_ref.worker_id,
error = %retain_error,
"failed to persist pending restore reconciliation marker"
);
}
Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_backend_outcome_unknown",
"Worker restore backend outcome is uncertain; reread or retry the Worker restore",
))
} }
} }
} }
@@ -2469,27 +2636,83 @@ impl Runtime {
worker_state: protocol::WorkerStateSnapshot, worker_state: protocol::WorkerStateSnapshot,
status: WorkerStatus, status: WorkerStatus,
working_directory: Option<CatalogWorkingDirectoryStatus>, working_directory: Option<CatalogWorkingDirectoryStatus>,
) -> Result<(), RuntimeError> { ) -> Result<WorkerDetail, RuntimeError> {
let mut state = self.lock()?; let mut state = self.lock()?;
state.ensure_worker_ref(worker_ref)?; state.ensure_worker_ref(worker_ref)?;
{ let mut candidate = state.worker(worker_ref)?.clone();
let worker = state.worker_mut(worker_ref)?; if candidate.execution_handle.is_some() {
if worker.execution_handle.is_some() { return Err(RuntimeError::InvalidRequest(format!(
return Err(RuntimeError::InvalidRequest(format!( "worker {} already has a current execution",
"worker {} already has a current execution", worker_ref.worker_id
worker_ref.worker_id )));
)));
}
worker.execution_handle = Some(handle);
worker.execution_metadata_available = true;
worker.execution_bound = true;
worker.status = status;
let _ = worker.apply_worker_state(&worker_state);
worker.restore_intent = restore_intent_for_status(worker.status);
worker.working_directory = working_directory;
} }
candidate.execution_handle = Some(handle);
candidate.execution_metadata_available = true;
candidate.execution_bound = true;
candidate.status = status;
let _ = candidate.apply_worker_state(&worker_state);
candidate.restore_intent = restore_intent_for_status(candidate.status);
candidate.working_directory = working_directory;
// Validate the exact subscription projection before persistence. The
// subsequent publish uses the same pure projection while the Worker
// operation lock keeps this candidate stable.
let _subscription_projection = state.subscription_worker(&candidate)?;
// Persist the complete candidate before making it observable. A failed
// write therefore leaves both the in-memory catalog and subscriptions
// on the pre-restore record, so cleanup can produce a true rollback.
state.persist_worker_record(&candidate)?;
let detail = candidate.detail();
state.workers.insert(worker_ref.worker_id, candidate);
state.publish_worker_upsert(worker_ref.worker_id)?;
if let Err(error) = state.persist_runtime_snapshot() {
tracing::warn!(
worker_id = %worker_ref.worker_id,
error = %error,
"Worker restore committed to its canonical record but Runtime snapshot refresh failed"
);
}
Ok(detail)
}
fn retain_restore_execution_evidence(
&self,
worker_ref: &WorkerRef,
handle: WorkerExecutionHandle,
worker_state: protocol::WorkerStateSnapshot,
working_directory: Option<CatalogWorkingDirectoryStatus>,
) -> Result<(), RuntimeError> {
let mut state = self.lock()?;
let mut candidate = state.worker(worker_ref)?.clone();
candidate.execution_handle = Some(handle);
candidate.execution_metadata_available = true;
candidate.execution_bound = true;
candidate.status = WorkerStatus::Idle;
let _ = candidate.apply_worker_state(&worker_state);
candidate.restore_intent = WorkerRestoreIntent::Automatic;
candidate.working_directory = working_directory;
state.workers.insert(worker_ref.worker_id, candidate);
state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_worker(&worker_ref.worker_id)?;
Ok(())
}
fn retain_restore_reconciliation_pending(
&self,
worker_ref: &WorkerRef,
) -> Result<(), RuntimeError> {
let mut state = self.lock()?;
let mut candidate = state.worker(worker_ref)?.clone();
candidate.execution_handle = None;
candidate.execution_metadata_available = false;
candidate.execution_bound = true;
// Active-without-metadata is the durable pending-reconciliation marker:
// it prevents a false Stopped projection and blocks duplicate restore.
candidate.status = WorkerStatus::Idle;
candidate.restore_intent = WorkerRestoreIntent::Explicit;
state.workers.insert(worker_ref.worker_id, candidate);
state.publish_worker_upsert(worker_ref.worker_id)?; state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?; state.persist_worker(&worker_ref.worker_id)?;
Ok(()) Ok(())
} }
@@ -2513,18 +2736,6 @@ impl Runtime {
result, result,
}); });
} }
let mut state = self.lock()?;
let worker = state.worker_mut(worker_ref)?;
worker.execution_handle = None;
worker.execution_bound = false;
worker.status = WorkerStatus::Stopped;
worker.worker_state = None;
worker.restore_intent = WorkerRestoreIntent::Explicit;
worker.internal_workers.clear();
state.publish_worker_upsert(worker_ref.worker_id)?;
state.persist_runtime_snapshot()?;
state.persist_worker(&worker_ref.worker_id)?;
Ok(()) Ok(())
} }
@@ -2908,6 +3119,14 @@ impl RuntimeState {
Ok(()) Ok(())
} }
#[cfg(feature = "fs-store")]
fn persist_worker_record(&self, worker: &WorkerRecord) -> Result<(), RuntimeError> {
if let Some(store) = self.fs_store() {
store.write_worker_record(&worker.persisted_record())?;
}
Ok(())
}
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
fn persist_worker(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> { fn persist_worker(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> {
if let Some(store) = self.fs_store() { if let Some(store) = self.fs_store() {
@@ -2945,6 +3164,11 @@ impl RuntimeState {
Ok(()) Ok(())
} }
#[cfg(not(feature = "fs-store"))]
fn persist_worker_record(&self, _worker: &WorkerRecord) -> Result<(), RuntimeError> {
Ok(())
}
#[cfg(not(feature = "fs-store"))] #[cfg(not(feature = "fs-store"))]
fn persist_worker(&self, _worker_id: &WorkerId) -> Result<(), RuntimeError> { fn persist_worker(&self, _worker_id: &WorkerId) -> Result<(), RuntimeError> {
Ok(()) Ok(())
@@ -3689,7 +3913,7 @@ struct RemovedInternalWorkerActivity {
parent_session_id: Option<String>, parent_session_id: Option<String>,
} }
#[derive(Debug, Default)] #[derive(Debug, Default, Clone)]
struct InternalWorkerActivityProjection { struct InternalWorkerActivityProjection {
workers: BTreeMap<String, InternalWorkerActivity>, workers: BTreeMap<String, InternalWorkerActivity>,
removed_workers: BTreeMap<String, RemovedInternalWorkerActivity>, removed_workers: BTreeMap<String, RemovedInternalWorkerActivity>,
@@ -3708,7 +3932,7 @@ impl InternalWorkerActivityProjection {
} }
} }
#[derive(Debug)] #[derive(Debug, Clone)]
struct WorkerRecord { struct WorkerRecord {
worker_ref: WorkerRef, worker_ref: WorkerRef,
worker_id: WorkerId, worker_id: WorkerId,
@@ -4998,6 +5222,7 @@ mod tests {
stop_gate: Mutex<Option<Arc<RestoreGate>>>, stop_gate: Mutex<Option<Arc<RestoreGate>>>,
stop_count: Mutex<u64>, stop_count: Mutex<u64>,
restore_result: Mutex<Option<WorkerExecutionSpawnResult>>, restore_result: Mutex<Option<WorkerExecutionSpawnResult>>,
preflight_restore_result: Mutex<Option<WorkerExecutionResult>>,
restore_gate: Mutex<Option<Arc<RestoreGate>>>, restore_gate: Mutex<Option<Arc<RestoreGate>>>,
restore_count: Mutex<u64>, restore_count: Mutex<u64>,
config_bundles: Mutex<Vec<Option<ConfigBundle>>>, config_bundles: Mutex<Vec<Option<ConfigBundle>>>,
@@ -5117,6 +5342,16 @@ mod tests {
} }
} }
fn preflight_restore(
&self,
_request: &WorkerExecutionRestoreRequest,
) -> Result<(), WorkerExecutionResult> {
match self.preflight_restore_result.lock().unwrap().clone() {
Some(result) => Err(result),
None => Ok(()),
}
}
fn restore_worker( fn restore_worker(
&self, &self,
request: WorkerExecutionRestoreRequest, request: WorkerExecutionRestoreRequest,
@@ -7229,6 +7464,69 @@ mod tests {
let _ = std::fs::remove_dir_all(root); let _ = std::fs::remove_dir_all(root);
} }
#[test]
fn restore_preflight_rejection_is_typed_and_side_effect_free() {
let (runtime, backend) = runtime_and_backend();
runtime.store_config_bundle(test_bundle()).unwrap();
let created = runtime
.create_worker(task_request("restore preflight"))
.unwrap();
runtime.stop_worker(&created.worker_ref, None).unwrap();
let before = runtime.worker_detail(&created.worker_ref).unwrap();
backend.preflight_restore_result.lock().unwrap().replace(
WorkerExecutionResult::rejected(
WorkerExecutionOperation::Restore,
"profile is no longer permitted",
),
);
let result = runtime
.restore_worker_operation(&created.worker_ref)
.unwrap();
assert_eq!(result.state, WorkerRestoreState::Rejected);
assert!(result.worker.is_none());
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
assert_eq!(runtime.worker_detail(&created.worker_ref).unwrap(), before);
}
#[test]
fn uncertain_restore_backend_outcome_requires_reconciliation() {
let (runtime, backend) = runtime_and_backend();
runtime.store_config_bundle(test_bundle()).unwrap();
let created = runtime
.create_worker(task_request("restore reconciliation"))
.unwrap();
runtime.stop_worker(&created.worker_ref, None).unwrap();
backend.restore_result.lock().unwrap().replace(
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::errored(
WorkerExecutionOperation::Restore,
"controller handshake outcome was not observable",
),
handle: None,
worker_state: None,
working_directory: None,
},
);
let result = runtime
.restore_worker_operation(&created.worker_ref)
.unwrap();
assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired);
assert!(result.worker.is_none());
assert_eq!(
runtime.worker_detail(&created.worker_ref).unwrap().status,
WorkerStatus::Idle
);
let retry = runtime
.restore_worker_operation(&created.worker_ref)
.unwrap();
assert_eq!(retry.state, WorkerRestoreState::ReconciliationRequired);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
}
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
#[test] #[test]
fn fs_store_automatically_restores_every_active_lifecycle_state() { fn fs_store_automatically_restores_every_active_lifecycle_state() {
@@ -7564,7 +7862,7 @@ mod tests {
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
#[test] #[test]
fn fs_store_stops_worker_and_reports_when_execution_restore_fails() { fn fs_store_marks_worker_for_reconciliation_when_execution_restore_is_uncertain() {
let root = fs_store_root("execution-restore-failed"); let root = fs_store_root("execution-restore-failed");
let runtime = Runtime::with_fs_store_and_execution_backend( let runtime = Runtime::with_fs_store_and_execution_backend(
crate::fs_store::FsRuntimeStoreOptions { crate::fs_store::FsRuntimeStoreOptions {
@@ -7598,17 +7896,13 @@ mod tests {
assert_eq!(*restoring_backend.restore_count.lock().unwrap(), 1); assert_eq!(*restoring_backend.restore_count.lock().unwrap(), 1);
let restored_worker = restored.worker_detail(&worker.worker_ref).unwrap(); let restored_worker = restored.worker_detail(&worker.worker_ref).unwrap();
assert_eq!(restored_worker.status, WorkerStatus::Stopped); assert_eq!(restored_worker.status, WorkerStatus::Idle);
assert!( assert!(!restored_worker.execution_metadata_available);
restored let retry = restored
.diagnostics() .restore_worker_operation(&worker.worker_ref)
.unwrap() .unwrap();
.iter() assert_eq!(retry.state, WorkerRestoreState::ReconciliationRequired);
.any( assert_eq!(*restoring_backend.restore_count.lock().unwrap(), 1);
|diagnostic| diagnostic.code == "worker_execution_restore_failed"
&& diagnostic.worker_ref.as_ref() == Some(&worker.worker_ref)
)
);
let err = restored let err = restored
.send_input( .send_input(
&worker.worker_ref, &worker.worker_ref,
+109 -28
View File
@@ -1666,15 +1666,34 @@ where
}) { }) {
Ok(task) => task, Ok(task) => task,
Err(message) => { Err(message) => {
let cleanup = self let cleanup = self.cleanup_unconnected_controller(
.cleanup_unconnected_controller(&handle, &shutdown, &tasks, &worker_state) &handle,
.err() &shutdown,
.map(|cleanup| format!("; controller cleanup failed: {cleanup}")) &tasks,
.unwrap_or_default(); &worker_state,
return WorkerExecutionSpawnResult::Errored(WorkerExecutionResult::errored( );
let result = WorkerExecutionResult::errored(
operation, operation,
format!("{message}{cleanup}"), match &cleanup {
)); Ok(()) => message,
Err(cleanup) => {
format!("{message}; controller cleanup failed: {cleanup}")
}
},
);
return if operation == WorkerExecutionOperation::Restore {
match cleanup {
Ok(()) => WorkerExecutionSpawnResult::RolledBack(result),
Err(_) => WorkerExecutionSpawnResult::ReconciliationRequired {
result,
handle: None,
worker_state: None,
working_directory: None,
},
}
} else {
WorkerExecutionSpawnResult::Errored(result)
};
} }
}; };
tasks.push("protocol bridge", bridge_task, true); tasks.push("protocol bridge", bridge_task, true);
@@ -1687,28 +1706,66 @@ where
let mut workers = match self.workers.lock() { let mut workers = match self.workers.lock() {
Ok(workers) => workers, Ok(workers) => workers,
Err(_) => { Err(_) => {
let cleanup = self let cleanup = self.cleanup_unconnected_controller(
.cleanup_unconnected_controller(&handle, &shutdown, &tasks, &worker_state) &handle,
.err() &shutdown,
.map(|cleanup| format!("; controller cleanup failed: {cleanup}")) &tasks,
.unwrap_or_default(); &worker_state,
return WorkerExecutionSpawnResult::Errored(WorkerExecutionResult::errored( );
let result = WorkerExecutionResult::errored(
operation, operation,
format!("worker adapter registry lock is poisoned{cleanup}"), match &cleanup {
)); Ok(()) => "worker adapter registry lock is poisoned".to_string(),
Err(cleanup) => format!(
"worker adapter registry lock is poisoned; controller cleanup failed: {cleanup}"
),
},
);
return if operation == WorkerExecutionOperation::Restore {
match cleanup {
Ok(()) => WorkerExecutionSpawnResult::RolledBack(result),
Err(_) => WorkerExecutionSpawnResult::ReconciliationRequired {
result,
handle: None,
worker_state: None,
working_directory: None,
},
}
} else {
WorkerExecutionSpawnResult::Errored(result)
};
} }
}; };
if workers.contains_key(&worker_ref) { if workers.contains_key(&worker_ref) {
drop(workers); drop(workers);
let cleanup = self let cleanup = self.cleanup_unconnected_controller(
.cleanup_unconnected_controller(&handle, &shutdown, &tasks, &worker_state) &handle,
.err() &shutdown,
.map(|cleanup| format!("; controller cleanup failed: {cleanup}")) &tasks,
.unwrap_or_default(); &worker_state,
return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::busy( );
let result = WorkerExecutionResult::busy(
operation, operation,
format!("Worker is already connected to execution backend{cleanup}"), match &cleanup {
)); Ok(()) => "Worker is already connected to execution backend".to_string(),
Err(cleanup) => format!(
"Worker is already connected to execution backend; controller cleanup failed: {cleanup}"
),
},
);
return if operation == WorkerExecutionOperation::Restore {
match cleanup {
Ok(()) => WorkerExecutionSpawnResult::RolledBack(result),
Err(_) => WorkerExecutionSpawnResult::ReconciliationRequired {
result,
handle: None,
worker_state: None,
working_directory: None,
},
}
} else {
WorkerExecutionSpawnResult::Rejected(result)
};
} }
let connected_worker_state = worker_state let connected_worker_state = worker_state
.read() .read()
@@ -1995,6 +2052,25 @@ where
) )
} }
fn preflight_restore(
&self,
request: &WorkerExecutionRestoreRequest,
) -> Result<(), WorkerExecutionResult> {
let workers = self.workers.lock().map_err(|_| {
WorkerExecutionResult::errored(
WorkerExecutionOperation::Restore,
"worker adapter registry lock is poisoned",
)
})?;
if workers.contains_key(&request.worker_ref) {
return Err(WorkerExecutionResult::busy(
WorkerExecutionOperation::Restore,
"Worker is already connected to execution backend",
));
}
Ok(())
}
fn restore_worker( fn restore_worker(
&self, &self,
mut request: WorkerExecutionRestoreRequest, mut request: WorkerExecutionRestoreRequest,
@@ -2076,10 +2152,15 @@ where
let controller = match restore_result { let controller = match restore_result {
Ok(controller) => controller, Ok(controller) => controller,
Err(message) => { Err(message) => {
return WorkerExecutionSpawnResult::Errored(WorkerExecutionResult::errored( return WorkerExecutionSpawnResult::ReconciliationRequired {
WorkerExecutionOperation::Restore, result: WorkerExecutionResult::errored(
message, WorkerExecutionOperation::Restore,
)); message,
),
handle: None,
worker_state: None,
working_directory: None,
};
} }
}; };
+58 -1
View File
@@ -2327,15 +2327,36 @@ pub enum WorkerOperationState {
Rejected, Rejected,
} }
/// Cross-layer outcome of an explicit Worker restore operation.
///
/// `Rejected` is reserved for read-only preflight failures. Once live restore
/// work starts, failure is either a confirmed `RolledBack` operation or a
/// `ReconciliationRequired` result whose commit state must be reread/retried.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum WorkerRestoreState {
Accepted,
Rejected,
RolledBack,
ReconciliationRequired,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct WorkerRestoreResult { pub struct WorkerRestoreResult {
pub state: WorkerOperationState, pub state: WorkerRestoreState,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
pub worker: Option<WorkerSummary>, pub worker: Option<WorkerSummary>,
#[serde(default)] #[serde(default)]
pub diagnostics: Vec<Diagnostic>, pub diagnostics: Vec<Diagnostic>,
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(deny_unknown_fields)]
pub struct WorkerRestoreResponse { pub struct WorkerRestoreResponse {
pub workspace_id: String, pub workspace_id: String,
pub runtime_id: String, pub runtime_id: String,
@@ -3482,6 +3503,42 @@ mod workdir_typescript_tests {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn worker_restore_state_round_trips_and_rejects_unknown_variants() {
for (state, wire) in [
(WorkerRestoreState::Accepted, "accepted"),
(WorkerRestoreState::Rejected, "rejected"),
(WorkerRestoreState::RolledBack, "rolled_back"),
(
WorkerRestoreState::ReconciliationRequired,
"reconciliation_required",
),
] {
assert_eq!(
serde_json::to_value(state).unwrap(),
serde_json::json!(wire)
);
assert_eq!(
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(wire)).unwrap(),
state
);
}
assert!(
serde_json::from_value::<WorkerRestoreState>(serde_json::json!("unknown")).is_err()
);
}
#[test]
fn worker_restore_result_rejects_unknown_fields() {
let result = serde_json::from_value::<WorkerRestoreResult>(serde_json::json!({
"state": "rejected",
"worker": null,
"diagnostics": [],
"unexpected": true
}));
assert!(result.is_err());
}
#[test] #[test]
fn historical_http_repository_source_kind_decodes_as_invalid_evidence() { fn historical_http_repository_source_kind_decodes_as_invalid_evidence() {
let source: RepositorySource = serde_json::from_value(serde_json::json!({ let source: RepositorySource = serde_json::from_value(serde_json::json!({
+71 -17
View File
@@ -52,6 +52,7 @@ use worker_runtime::http_server::{
RuntimeHttpWorkerCompletionsResponse, RuntimeHttpWorkerDeleteResponse, RuntimeHttpWorkerCompletionsResponse, RuntimeHttpWorkerDeleteResponse,
RuntimeHttpWorkerInputResponse, RuntimeHttpWorkerLifecycleRequest, RuntimeHttpWorkerInputResponse, RuntimeHttpWorkerLifecycleRequest,
RuntimeHttpWorkerLifecycleResponse, RuntimeHttpWorkerResponse, RuntimeHttpWorkerLifecycleResponse, RuntimeHttpWorkerResponse,
RuntimeHttpWorkerRestoreResponse,
RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse, RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse,
RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse, RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse,
RuntimeHttpWorkspacePromptProjectionRequest, RuntimeHttpWorkspacePromptProjectionResponse, RuntimeHttpWorkspacePromptProjectionRequest, RuntimeHttpWorkspacePromptProjectionResponse,
@@ -378,8 +379,9 @@ pub(crate) fn workspace_worker_summary(
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct WorkerRestoreResult { pub struct WorkerRestoreResult {
pub state: WorkerOperationState, pub state: workspace_api::WorkerRestoreState,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub worker: Option<WorkerSummary>, pub worker: Option<WorkerSummary>,
pub diagnostics: Vec<RuntimeDiagnostic>, pub diagnostics: Vec<RuntimeDiagnostic>,
@@ -882,7 +884,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult { fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
WorkerRestoreResult { WorkerRestoreResult {
state: WorkerOperationState::Unsupported, state: workspace_api::WorkerRestoreState::Rejected,
worker: None, worker: None,
diagnostics: vec![diagnostic( diagnostics: vec![diagnostic(
"worker_restore_unsupported", "worker_restore_unsupported",
@@ -2336,7 +2338,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult { fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
let Some(worker_ref) = self.worker_ref(worker_id) else { let Some(worker_ref) = self.worker_ref(worker_id) else {
return WorkerRestoreResult { return WorkerRestoreResult {
state: WorkerOperationState::Rejected, state: workspace_api::WorkerRestoreState::Rejected,
worker: None, worker: None,
diagnostics: vec![diagnostic( diagnostics: vec![diagnostic(
"embedded_worker_id_invalid", "embedded_worker_id_invalid",
@@ -2345,14 +2347,24 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
)], )],
}; };
}; };
match self.runtime.restore_worker(&worker_ref) { match self.runtime.restore_worker_operation(&worker_ref) {
Ok(detail) => WorkerRestoreResult { Ok(result) => {
state: WorkerOperationState::Accepted, let diagnostics = match (result.reason_code, result.message) {
worker: Some(self.map_worker_detail(detail)), (Some(code), Some(message)) => vec![diagnostic(
diagnostics: Vec::new(), code,
}, DiagnosticSeverity::Warning,
message,
)],
_ => Vec::new(),
};
WorkerRestoreResult {
state: result.state,
worker: result.worker.map(|detail| self.map_worker_detail(detail)),
diagnostics,
}
}
Err(err) => WorkerRestoreResult { Err(err) => WorkerRestoreResult {
state: WorkerOperationState::Rejected, state: workspace_api::WorkerRestoreState::ReconciliationRequired,
worker: None, worker: None,
diagnostics: vec![embedded_runtime_diagnostic(&err)], diagnostics: vec![embedded_runtime_diagnostic(&err)],
}, },
@@ -4164,17 +4176,27 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
} }
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult { fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
match self.post_json::<_, RuntimeHttpWorkerResponse>( match self.post_json::<_, RuntimeHttpWorkerRestoreResponse>(
&format!("/v1/workers/{worker_id}/restore"), &format!("/v1/workers/{worker_id}/restore"),
&serde_json::json!({}), &serde_json::json!({}),
) { ) {
Ok(response) => WorkerRestoreResult { Ok(response) => {
state: WorkerOperationState::Accepted, let diagnostics = match (response.reason_code, response.message) {
worker: Some(self.map_worker_detail(response.worker)), (Some(code), Some(message)) => {
diagnostics: Vec::new(), vec![diagnostic(code, DiagnosticSeverity::Warning, message)]
}, }
_ => Vec::new(),
};
WorkerRestoreResult {
state: response.state,
worker: response.worker.map(|worker| self.map_worker_detail(worker)),
diagnostics,
}
}
Err(diagnostic) => WorkerRestoreResult { Err(diagnostic) => WorkerRestoreResult {
state: WorkerOperationState::Rejected, // A transport/protocol failure cannot prove that the Runtime
// rejected before side effects. Preserve uncertainty.
state: workspace_api::WorkerRestoreState::ReconciliationRequired,
worker: None, worker: None,
diagnostics: vec![diagnostic], diagnostics: vec![diagnostic],
}, },
@@ -5428,6 +5450,38 @@ mod tests {
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::thread; use std::thread;
#[test]
fn remote_restore_transport_failure_requires_reconciliation() {
let provider = RemoteWorkerRuntime::new(
RemoteRuntimeConfig {
runtime_id: "runtime-remote".to_string(),
workspace_id: Some("workspace-a".to_string()),
display_name: "Remote Runtime".to_string(),
base_url: "http://127.0.0.1:9".to_string(),
bearer_token: None,
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "linux".to_string(),
cached_arch: "x86_64".to_string(),
cached_status: "ready".to_string(),
timeout: Duration::from_millis(100),
},
"workspace-a".to_string(),
"http://127.0.0.1:3000".to_string(),
)
.unwrap();
let result = provider.restore_worker("worker-transport-uncertain");
assert_eq!(
result.state,
workspace_api::WorkerRestoreState::ReconciliationRequired
);
assert!(result.worker.is_none());
assert_eq!(result.diagnostics.len(), 1);
}
#[test] #[test]
fn embedded_delete_persistence_failure_diagnostic_is_bounded_and_path_free() { fn embedded_delete_persistence_failure_diagnostic_is_bounded_and_path_free() {
let diagnostic = let diagnostic =
+8 -10
View File
@@ -2814,7 +2814,7 @@ impl WorkspaceApi {
.map_err(|error| error.into_error())?; .map_err(|error| error.into_error())?;
if binding.state != WorkerOperationState::Accepted { if binding.state != WorkerOperationState::Accepted {
return Ok(WorkerRestoreResult { return Ok(WorkerRestoreResult {
state: binding.state, state: workspace_api::WorkerRestoreState::Rejected,
worker: binding.worker, worker: binding.worker,
diagnostics: binding.diagnostics, diagnostics: binding.diagnostics,
}); });
@@ -10422,7 +10422,7 @@ async fn scoped_start_workspace_orchestrator(
.runtime .runtime
.restore_worker(&existing.worker) .restore_worker(&existing.worker)
.map_err(|error| error.into_error())?; .map_err(|error| error.into_error())?;
if restored.state != WorkerOperationState::Accepted { if restored.state != workspace_api::WorkerRestoreState::Accepted {
return Err(ApiError::with_diagnostics( return Err(ApiError::with_diagnostics(
Error::RuntimeOperationFailed { Error::RuntimeOperationFailed {
runtime_id: existing.worker.runtime_id.clone(), runtime_id: existing.worker.runtime_id.clone(),
@@ -12501,7 +12501,7 @@ async fn scoped_restore_runtime_worker(
runtime_id: runtime_id.clone(), runtime_id: runtime_id.clone(),
worker_id: worker_id.clone(), worker_id: worker_id.clone(),
result: workspace_api::WorkerRestoreResult { result: workspace_api::WorkerRestoreResult {
state: workspace_api::WorkerOperationState::Accepted, state: workspace_api::WorkerRestoreState::Accepted,
worker: Some(worker), worker: Some(worker),
diagnostics: Vec::new(), diagnostics: Vec::new(),
}, },
@@ -15213,7 +15213,7 @@ async fn restore_runtime_worker(
runtime_id: runtime_id.clone(), runtime_id: runtime_id.clone(),
worker_id: worker_id.clone(), worker_id: worker_id.clone(),
result: workspace_api::WorkerRestoreResult { result: workspace_api::WorkerRestoreResult {
state: result.state.into(), state: result.state,
worker: projected_worker, worker: projected_worker,
diagnostics: result.diagnostics.into_iter().map(Into::into).collect(), diagnostics: result.diagnostics.into_iter().map(Into::into).collect(),
}, },
@@ -20722,7 +20722,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn production_profile_backend_rejects_unrecoverable_pending_orchestrator_restore() { async fn production_profile_backend_reports_uncertain_pending_orchestrator_restore() {
let workspace = tempfile::tempdir().unwrap(); let workspace = tempfile::tempdir().unwrap();
init_clean_git_workspace(workspace.path()); init_clean_git_workspace(workspace.path());
let config = test_server_config(workspace.path()); let config = test_server_config(workspace.path());
@@ -20772,11 +20772,9 @@ mod tests {
}), }),
) )
.await .await
.expect_err("pending Workspace Orchestrator restore without durable Prompt must fail"); .expect_err("pending Workspace Orchestrator restore without durable Prompt must require reconciliation");
assert!( assert!(
format!("{error:?}").contains( format!("{error:?}").contains("worker_restore_reconciliation_required"),
"pending Workspace Worker restore requires operation-owned launch material"
),
"unexpected restore error: {error:?}" "unexpected restore error: {error:?}"
); );
} }
@@ -25277,7 +25275,7 @@ mod tests {
assert_eq!(retried_restore.worker_id, first_worker.worker.worker_id); assert_eq!(retried_restore.worker_id, first_worker.worker.worker_id);
assert_eq!( assert_eq!(
retried_restore.result.state, retried_restore.result.state,
workspace_api::WorkerOperationState::Accepted workspace_api::WorkerRestoreState::Accepted
); );
let restored_assignment = api let restored_assignment = api
.store .store
@@ -112,6 +112,19 @@ export type Worker = {
export type WorkerOperationState = "accepted" | "unsupported" | "rejected"; export type WorkerOperationState = "accepted" | "unsupported" | "rejected";
/** Typed result of restore across Runtime, Workspace, Web, and TUI clients. */
export type WorkerRestoreState =
| "accepted"
| "rejected"
| "rolled_back"
| "reconciliation_required";
export type WorkerRestoreResult = {
state: WorkerRestoreState;
worker?: WorkerSummary | null;
diagnostics: Diagnostic[];
};
export type WorkerLaunchRuntimeOption = SharedWorkerLaunchRuntimeOption; export type WorkerLaunchRuntimeOption = SharedWorkerLaunchRuntimeOption;
export type WorkerLaunchProfileCandidate = SharedWorkerLaunchProfileCandidate; export type WorkerLaunchProfileCandidate = SharedWorkerLaunchProfileCandidate;
export type WorkingDirectoryRepositoryOption = export type WorkingDirectoryRepositoryOption =