From 860252c22dc47983fc532143ad56582cad1492e2 Mon Sep 17 00:00:00 2001 From: Hare Date: Fri, 2 Oct 2026 14:00:04 +0900 Subject: [PATCH] fix: report actual worker execution unavailability --- crates/worker-runtime/src/error.rs | 2 +- crates/worker-runtime/src/runtime.rs | 70 ++++++++++++++++++++------- crates/workspace-server/src/hosts.rs | 71 ++++++++++++++++++++++++++-- 3 files changed, 121 insertions(+), 22 deletions(-) diff --git a/crates/worker-runtime/src/error.rs b/crates/worker-runtime/src/error.rs index c0f108386..bbbea4435 100644 --- a/crates/worker-runtime/src/error.rs +++ b/crates/worker-runtime/src/error.rs @@ -15,7 +15,7 @@ pub enum RuntimeError { #[error("worker {worker_id} was not found")] WorkerNotFound { worker_id: WorkerId }, - #[error("worker {worker_id} has no execution backend: {message}")] + #[error("worker {worker_id} execution is unavailable: {message}")] WorkerExecutionUnavailable { worker_id: WorkerId, message: String, diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index f16d5f603..d46b38847 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1425,9 +1425,9 @@ impl Runtime { let previous_workspace_api = { let state = self.lock()?; let worker = state.worker(worker_ref)?; - if worker.has_pending_lifecycle_operation() { + if let Some(operation) = worker.pending_lifecycle_operation_name() { return Err(RuntimeError::InvalidRequest(format!( - "worker {} has pending restore reconciliation", + "worker {} has pending {operation} reconciliation", worker_ref.worker_id ))); } @@ -1453,9 +1453,9 @@ impl Runtime { { let mut state = self.lock()?; let worker = state.worker_mut(worker_ref)?; - if worker.has_pending_lifecycle_operation() { + if let Some(operation) = worker.pending_lifecycle_operation_name() { return Err(RuntimeError::InvalidRequest(format!( - "worker {} has pending restore reconciliation", + "worker {} has pending {operation} reconciliation", worker_ref.worker_id ))); } @@ -2052,10 +2052,10 @@ impl Runtime { fn ensure_worker_execution(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> { let state = self.lock()?; let worker = state.worker(worker_ref)?; - if worker.has_pending_lifecycle_operation() { + if let Some(operation) = worker.pending_lifecycle_operation_name() { return Err(RuntimeError::WorkerExecutionUnavailable { worker_id: worker_ref.worker_id, - message: "worker restore reconciliation is pending".to_string(), + message: format!("worker {operation} reconciliation is pending"), }); } if worker.execution_handle.is_some() { @@ -2734,9 +2734,9 @@ impl Runtime { let state = self.lock()?; state.ensure_running()?; let worker = state.worker(worker_ref)?; - if worker.has_pending_lifecycle_operation() { + if let Some(operation) = worker.pending_lifecycle_operation_name() { return Err(RuntimeError::InvalidRequest(format!( - "worker {} has pending restore reconciliation", + "worker {} has pending {operation} reconciliation", worker_ref.worker_id ))); } @@ -2793,9 +2793,9 @@ impl Runtime { state.ensure_running()?; state.ensure_worker_ref(worker_ref)?; let worker = state.worker(worker_ref)?; - if worker.has_pending_lifecycle_operation() { + if let Some(operation) = worker.pending_lifecycle_operation_name() { return Err(RuntimeError::InvalidRequest(format!( - "worker {} has pending restore reconciliation and cannot be deleted", + "worker {} has pending {operation} reconciliation and cannot be deleted", worker_ref.worker_id ))); } @@ -2834,9 +2834,9 @@ impl Runtime { state.ensure_running()?; state.ensure_worker_ref(worker_ref)?; let worker = state.worker(worker_ref)?; - if worker.has_pending_lifecycle_operation() { + if let Some(operation) = worker.pending_lifecycle_operation_name() { return Err(RuntimeError::InvalidRequest(format!( - "worker {} still has pending restore reconciliation", + "worker {} still has pending {operation} reconciliation", worker_ref.worker_id ))); } @@ -3619,10 +3619,10 @@ impl Runtime { "Worker retention requires a stopped Worker".to_string(), )); } - if worker.has_pending_lifecycle_operation() { - return Err(RuntimeError::InvalidRequest( - "Worker retention cannot remove pending restore reconciliation".to_string(), - )); + if let Some(operation) = worker.pending_lifecycle_operation_name() { + return Err(RuntimeError::InvalidRequest(format!( + "Worker retention cannot remove pending {operation} reconciliation" + ))); } let result = provider.execute(request)?; state.workers.remove(&request.worker_id); @@ -4801,8 +4801,18 @@ impl WorkerRecord { self.worker_state = Some(incoming.clone()); } + fn pending_lifecycle_operation_name(&self) -> Option<&'static str> { + if self.pending_stop.is_some() { + Some("stop") + } else if self.pending_restore.is_some() { + Some("restore") + } else { + None + } + } + fn has_pending_lifecycle_operation(&self) -> bool { - self.pending_restore.is_some() || self.pending_stop.is_some() + self.pending_lifecycle_operation_name().is_some() } fn belongs_to_workspace(&self, workspace_id: &str) -> bool { @@ -7418,6 +7428,30 @@ mod tests { assert!(runtime.worker_operations.lock().unwrap().is_empty()); } + fn assert_pending_lifecycle_errors(runtime: &Runtime, worker_ref: &WorkerRef, operation: &str) { + let input_error = runtime + .send_input(worker_ref, WorkerInput::user("blocked while pending")) + .unwrap_err(); + assert_eq!( + input_error.to_string(), + format!( + "worker {} execution is unavailable: worker {operation} reconciliation is pending", + worker_ref.worker_id + ) + ); + for error in [ + runtime.cancel_worker(worker_ref, None).unwrap_err(), + runtime.delete_worker(worker_ref).unwrap_err(), + ] { + assert!( + error + .to_string() + .contains(&format!("pending {operation} reconciliation")), + "{error}" + ); + } + } + #[test] fn failed_stop_cleanup_retains_execution_for_retry() { let (runtime, backend) = runtime_and_backend(); @@ -7454,6 +7488,7 @@ mod tests { backend.stop_operation_ids.lock().unwrap().as_slice(), &[operation_id] ); + assert_pending_lifecycle_errors(&runtime, &created.worker_ref, "stop"); let stopped = runtime.stop_worker(&created.worker_ref, None).unwrap(); assert_eq!(stopped.status, WorkerStatus::Stopped); @@ -9133,6 +9168,7 @@ mod tests { .pending_restore .unwrap() .operation_id; + assert_pending_lifecycle_errors(&runtime, &created.worker_ref, "restore"); let delete_error = runtime.delete_worker(&created.worker_ref).unwrap_err(); assert!( delete_error diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index aa903a20f..c89fc2909 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -5522,6 +5522,10 @@ fn sanitize_embedded_execution_message( ) -> String { let summary = format!("Embedded Worker execution backend rejected {operation:?} with {outcome:?}"); + sanitize_embedded_execution_detail(&summary, message) +} + +fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String { let mut redact_next = false; let detail = message .split_whitespace() @@ -5558,7 +5562,7 @@ fn sanitize_embedded_execution_message( .join(" "); let detail = detail.trim(); if detail.is_empty() { - return summary; + return summary.to_string(); } let truncated = detail.chars().count() > 512; let mut detail = detail.chars().take(512).collect::(); @@ -5580,11 +5584,18 @@ fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnosti HostDiagnosticSeverity::Warning, "Embedded Runtime worker was not found".to_string(), ), - EmbeddedRuntimeError::WorkerExecutionUnavailable { .. } - | EmbeddedRuntimeError::ExecutionBackendUnavailable { .. } => diagnostic( + EmbeddedRuntimeError::WorkerExecutionUnavailable { message, .. } => diagnostic( "embedded_worker_execution_unavailable", HostDiagnosticSeverity::Warning, - "Embedded Worker has no execution backend attached".to_string(), + sanitize_embedded_execution_detail("Embedded Worker execution is unavailable", message), + ), + EmbeddedRuntimeError::ExecutionBackendUnavailable { message } => diagnostic( + "embedded_worker_execution_unavailable", + HostDiagnosticSeverity::Warning, + sanitize_embedded_execution_detail( + "Embedded Runtime execution backend is unavailable", + message, + ), ), EmbeddedRuntimeError::WorkerExecutionRejected { operation, @@ -6055,6 +6066,58 @@ mod tests { assert_eq!(result.diagnostics.len(), 1); } + #[test] + fn embedded_execution_unavailable_preserves_the_actual_reason() { + for reason in [ + "worker stop reconciliation is pending", + "worker restore reconciliation is pending", + "stopped worker requires an explicit restore", + "worker has no live execution handle", + "runtime has no execution backend", + ] { + let diagnostic = + embedded_runtime_diagnostic(&EmbeddedRuntimeError::WorkerExecutionUnavailable { + worker_id: EmbeddedWorkerId::now_v7(), + message: reason.to_string(), + }); + assert_eq!(diagnostic.code, "embedded_worker_execution_unavailable"); + assert_eq!(diagnostic.severity, HostDiagnosticSeverity::Warning); + assert_eq!( + diagnostic.message, + format!("Embedded Worker execution is unavailable: {reason}") + ); + } + let diagnostic = + embedded_runtime_diagnostic(&EmbeddedRuntimeError::ExecutionBackendUnavailable { + message: "Runtime has no execution backend".to_string(), + }); + assert_eq!( + diagnostic.message, + "Embedded Runtime execution backend is unavailable: Runtime has no execution backend" + ); + } + + #[test] + fn embedded_execution_unavailable_detail_remains_bounded_and_redacted() { + let diagnostic = embedded_runtime_diagnostic( + &EmbeddedRuntimeError::WorkerExecutionUnavailable { + worker_id: EmbeddedWorkerId::now_v7(), + message: format!( + "worker stop reconciliation is pending /private/runtime Bearer sensitive-value {}", + "x".repeat(600) + ), + }, + ); + assert!( + diagnostic + .message + .contains("stop reconciliation is pending") + ); + assert!(!diagnostic.message.contains("/private/runtime")); + assert!(!diagnostic.message.contains("sensitive-value")); + assert!(diagnostic.message.chars().count() < 600); + } + #[test] fn embedded_delete_persistence_failure_diagnostic_is_bounded_and_path_free() { let diagnostic =