Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

Implement transactional Worker restore outcomes

Browse Source
This commit is contained in:
Hare
2026-09-18 00:08:00 +09:00
parent 5ae1bab4ee
commit 1d7cd9339c
11 changed files with 739 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/hosts.rs
+71 -17
View File
@@ -52,6 +52,7 @@ use worker_runtime::http_server::{
RuntimeHttpWorkerCompletionsResponse, RuntimeHttpWorkerDeleteResponse,
RuntimeHttpWorkerInputResponse, RuntimeHttpWorkerLifecycleRequest,
RuntimeHttpWorkerLifecycleResponse, RuntimeHttpWorkerResponse,
RuntimeHttpWorkerRestoreResponse,
RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse,
RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse,
RuntimeHttpWorkspacePromptProjectionRequest, RuntimeHttpWorkspacePromptProjectionResponse,
@@ -378,8 +379,9 @@ pub(crate) fn workspace_worker_summary(
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct WorkerRestoreResult {
pub state: WorkerOperationState,
pub state: workspace_api::WorkerRestoreState,
#[serde(skip_serializing_if = "Option::is_none")]
pub worker: Option<WorkerSummary>,
pub diagnostics: Vec<RuntimeDiagnostic>,
@@ -882,7 +884,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync {
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
WorkerRestoreResult {
state: WorkerOperationState::Unsupported,
state: workspace_api::WorkerRestoreState::Rejected,
worker: None,
diagnostics: vec![diagnostic(
"worker_restore_unsupported",
@@ -2336,7 +2338,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
let Some(worker_ref) = self.worker_ref(worker_id) else {
return WorkerRestoreResult {
state: WorkerOperationState::Rejected,
state: workspace_api::WorkerRestoreState::Rejected,
worker: None,
diagnostics: vec![diagnostic(
"embedded_worker_id_invalid",
@@ -2345,14 +2347,24 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime {
)],
};
};
match self.runtime.restore_worker(&worker_ref) {
Ok(detail) => WorkerRestoreResult {
state: WorkerOperationState::Accepted,
worker: Some(self.map_worker_detail(detail)),
diagnostics: Vec::new(),
},
match self.runtime.restore_worker_operation(&worker_ref) {
Ok(result) => {
let diagnostics = match (result.reason_code, result.message) {
(Some(code), Some(message)) => vec![diagnostic(
code,
DiagnosticSeverity::Warning,
message,
)],
_ => Vec::new(),
};
WorkerRestoreResult {
state: result.state,
worker: result.worker.map(|detail| self.map_worker_detail(detail)),
diagnostics,
}
}
Err(err) => WorkerRestoreResult {
state: WorkerOperationState::Rejected,
state: workspace_api::WorkerRestoreState::ReconciliationRequired,
worker: None,
diagnostics: vec![embedded_runtime_diagnostic(&err)],
},
@@ -4164,17 +4176,27 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
}
fn restore_worker(&self, worker_id: &str) -> WorkerRestoreResult {
match self.post_json::<_, RuntimeHttpWorkerResponse>(
match self.post_json::<_, RuntimeHttpWorkerRestoreResponse>(
&format!("/v1/workers/{worker_id}/restore"),
&serde_json::json!({}),
) {
Ok(response) => WorkerRestoreResult {
state: WorkerOperationState::Accepted,
worker: Some(self.map_worker_detail(response.worker)),
diagnostics: Vec::new(),
},
Ok(response) => {
let diagnostics = match (response.reason_code, response.message) {
(Some(code), Some(message)) => {
vec![diagnostic(code, DiagnosticSeverity::Warning, message)]
}
_ => Vec::new(),
};
WorkerRestoreResult {
state: response.state,
worker: response.worker.map(|worker| self.map_worker_detail(worker)),
diagnostics,
}
}
Err(diagnostic) => WorkerRestoreResult {
state: WorkerOperationState::Rejected,
// A transport/protocol failure cannot prove that the Runtime
// rejected before side effects. Preserve uncertainty.
state: workspace_api::WorkerRestoreState::ReconciliationRequired,
worker: None,
diagnostics: vec![diagnostic],
},
@@ -5428,6 +5450,38 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::thread;
#[test]
fn remote_restore_transport_failure_requires_reconciliation() {
let provider = RemoteWorkerRuntime::new(
RemoteRuntimeConfig {
runtime_id: "runtime-remote".to_string(),
workspace_id: Some("workspace-a".to_string()),
display_name: "Remote Runtime".to_string(),
base_url: "http://127.0.0.1:9".to_string(),
bearer_token: None,
workspace_authorization: None,
strict_public_egress: false,
cached_worker_creation_available: true,
cached_os: "linux".to_string(),
cached_arch: "x86_64".to_string(),
cached_status: "ready".to_string(),
timeout: Duration::from_millis(100),
},
"workspace-a".to_string(),
"http://127.0.0.1:3000".to_string(),
)
.unwrap();
let result = provider.restore_worker("worker-transport-uncertain");
assert_eq!(
result.state,
workspace_api::WorkerRestoreState::ReconciliationRequired
);
assert!(result.worker.is_none());
assert_eq!(result.diagnostics.len(), 1);
}
#[test]
fn embedded_delete_persistence_failure_diagnostic_is_bounded_and_path_free() {
let diagnostic =
crates/workspace-server/src/server.rs
+8 -10
View File
@@ -2814,7 +2814,7 @@ impl WorkspaceApi {
.map_err(|error| error.into_error())?;
if binding.state != WorkerOperationState::Accepted {
return Ok(WorkerRestoreResult {
state: binding.state,
state: workspace_api::WorkerRestoreState::Rejected,
worker: binding.worker,
diagnostics: binding.diagnostics,
});
@@ -10422,7 +10422,7 @@ async fn scoped_start_workspace_orchestrator(
.runtime
.restore_worker(&existing.worker)
.map_err(|error| error.into_error())?;
if restored.state != WorkerOperationState::Accepted {
if restored.state != workspace_api::WorkerRestoreState::Accepted {
return Err(ApiError::with_diagnostics(
Error::RuntimeOperationFailed {
runtime_id: existing.worker.runtime_id.clone(),
@@ -12501,7 +12501,7 @@ async fn scoped_restore_runtime_worker(
runtime_id: runtime_id.clone(),
worker_id: worker_id.clone(),
result: workspace_api::WorkerRestoreResult {
state: workspace_api::WorkerOperationState::Accepted,
state: workspace_api::WorkerRestoreState::Accepted,
worker: Some(worker),
diagnostics: Vec::new(),
},
@@ -15213,7 +15213,7 @@ async fn restore_runtime_worker(
runtime_id: runtime_id.clone(),
worker_id: worker_id.clone(),
result: workspace_api::WorkerRestoreResult {
state: result.state.into(),
state: result.state,
worker: projected_worker,
diagnostics: result.diagnostics.into_iter().map(Into::into).collect(),
},
@@ -20722,7 +20722,7 @@ mod tests {
}
#[tokio::test]
async fn production_profile_backend_rejects_unrecoverable_pending_orchestrator_restore() {
async fn production_profile_backend_reports_uncertain_pending_orchestrator_restore() {
let workspace = tempfile::tempdir().unwrap();
init_clean_git_workspace(workspace.path());
let config = test_server_config(workspace.path());
@@ -20772,11 +20772,9 @@ mod tests {
}),
)
.await
.expect_err("pending Workspace Orchestrator restore without durable Prompt must fail");
.expect_err("pending Workspace Orchestrator restore without durable Prompt must require reconciliation");
assert!(
format!("{error:?}").contains(
"pending Workspace Worker restore requires operation-owned launch material"
),
format!("{error:?}").contains("worker_restore_reconciliation_required"),
"unexpected restore error: {error:?}"
);
}
@@ -25277,7 +25275,7 @@ mod tests {
assert_eq!(retried_restore.worker_id, first_worker.worker.worker_id);
assert_eq!(
retried_restore.result.state,
workspace_api::WorkerOperationState::Accepted
workspace_api::WorkerRestoreState::Accepted
);
let restored_assignment = api
.store